From 1b2d27fddd99c2d58a2536b770765ad4f94bbd75 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 15 Feb 2018 06:22:46 +0530 Subject: [PATCH 001/129] Initial commit --- java/adapter/jdbc/pom.xml | 52 ++++++++ .../arrow/adapter/jdbc/JdbcToArrow.java | 68 ++++++++++ .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 122 ++++++++++++++++++ .../jdbc/src/test/resources/dbcp.properties | 18 +++ 4 files changed, 260 insertions(+) create mode 100644 java/adapter/jdbc/pom.xml create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java create mode 100644 java/adapter/jdbc/src/test/resources/dbcp.properties diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml new file mode 100644 index 00000000000..cee23b0bc1f --- /dev/null +++ b/java/adapter/jdbc/pom.xml @@ -0,0 +1,52 @@ + + + + + + 4.0.0 + org.apache.arrow.adapter.jdbc + arrow-jdbc + jar + 0.9.0-SNAPSHOT + Arrow JDBC Adapter + http://maven.apache.org + + + + org.apache.commons + commons-dbcp2 + 2.1 + + + + + org.apache.arrow + arrow-memory + 0.9.0-SNAPSHOT + + + + org.apache.arrow + arrow-vector + 0.9.0-SNAPSHOT + + + + junit + junit + 4.11 + test + + + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java new file mode 100644 index 00000000000..a60b59a7b9c --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.sql.*; + +/** + * + */ +public class JdbcToArrow { + + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * + * @param connection Database connection to be used. This metho will not close hte passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @return + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. + */ + public static VectorSchemaRoot convert(Connection connection, String query) throws SQLException { + + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + + Statement stmt = null; + ResultSet rs = null; + ResultSetMetaData rsmd = null; + try { + stmt = connection.createStatement(); + rs = stmt.executeQuery(query); + VectorSchemaRoot root = VectorSchemaRoot.create( + JdbcToArrowUtils.jdbcToArrowSchema(rs.getMetaData()), rootAllocator); + + } catch (SQLException exc) { + + } finally { + if (rs != null) { + rs.close(); + } + if (stmt != null) { + stmt.close(); + } + } + return null; + } + + + +} diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java new file mode 100644 index 00000000000..66fd5cef198 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import com.google.common.collect.ImmutableList; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; + +import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; + + +/** + * + */ +public class JdbcToArrowUtils { + + /** + * JDBC type Java type + CHAR String + VARCHAR String + LONGVARCHAR String + NUMERIC java.math.BigDecimal + DECIMAL java.math.BigDecimal + BIT boolean + TINYINT byte + SMALLINT short + INTEGER int + BIGINT long + REAL float + FLOAT double + DOUBLE double + BINARY byte[] + VARBINARY byte[] + LONGVARBINARY byte[] + DATE java.sql.Date + TIME java.sql.Time + TIMESTAMP java.sql.Timestamp + + * @param rsmd + * @return + * @throws SQLException + */ + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + ImmutableList.Builder fields = ImmutableList.builder(); + int columnCount = rsmd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + switch (rsmd.getColumnType(i)) { + case Types.ARRAY: + // no-op as of now + break; + case Types.BIGINT: + fields.add(new Field("long", FieldType.nullable(new ArrowType.Int(64, true)), null)); + break; + case Types.BINARY: case Types.LONGVARBINARY: case Types.VARBINARY: + break; + case Types.BIT: + break; + case Types.BLOB: + fields.add(new Field("binary", FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.BOOLEAN: + break; + case Types.CHAR: + break; + case Types.CLOB: + break; + case Types.DATE: + break; + case Types.DECIMAL: case Types.NUMERIC: + break; + case Types.DOUBLE: case Types.FLOAT: + fields.add(new Field("double", FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); + break; + case Types.INTEGER: + fields.add(new Field("int", FieldType.nullable(new ArrowType.Int(32, true)), null)); + break; + case Types.LONGVARCHAR: case Types.LONGNVARCHAR: case Types.VARCHAR: case Types.NVARCHAR: + break; + case Types.REAL: + break; + case Types.SMALLINT: + break; + case Types.TIME: + break; + case Types.TIMESTAMP: + break; + case Types.TINYINT: + break; + default: +// throw new TypeConversionNotSupportedException(); + // no-op + + } + } + + return new Schema(fields.build(), null); + } + + +} diff --git a/java/adapter/jdbc/src/test/resources/dbcp.properties b/java/adapter/jdbc/src/test/resources/dbcp.properties new file mode 100644 index 00000000000..aea0a132477 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/dbcp.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. The ASF licenses this file to +# You under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +# by applicable law or agreed to in writing, software distributed under the +# icense is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +driverClassName=org.h2.Driver +url=jdbc:h2:~/test +username=sa +password= + + + From c482b50b6b9e0532dc1294662a02fa177e1af9f6 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 19 Feb 2018 02:40:56 +0530 Subject: [PATCH 002/129] Added necessary code for JDBC to Arrow schema creation and Arrow vector objects creation. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 12 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 213 +++++++++++++++--- 2 files changed, 193 insertions(+), 32 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index a60b59a7b9c..cc9413a90e6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -37,21 +37,23 @@ public class JdbcToArrow { * @return * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. */ - public static VectorSchemaRoot convert(Connection connection, String query) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); Statement stmt = null; ResultSet rs = null; - ResultSetMetaData rsmd = null; try { stmt = connection.createStatement(); rs = stmt.executeQuery(query); + ResultSetMetaData rsmd = rs.getMetaData(); VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(rs.getMetaData()), rootAllocator); - - } catch (SQLException exc) { + JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); + JdbcToArrowUtils.jdbcToArrowVectors(rs, root); + } catch (Exception exc) { + // just throw it out after logging + throw exc; } finally { if (rs != null) { rs.close(); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 66fd5cef198..32ad3064ee4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -19,15 +19,20 @@ package org.apache.arrow.adapter.jdbc; import com.google.common.collect.ImmutableList; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -63,60 +68,214 @@ public class JdbcToArrowUtils { * @throws SQLException */ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + + assert rsmd != null; + ImmutableList.Builder fields = ImmutableList.builder(); int columnCount = rsmd.getColumnCount(); for (int i = 1; i <= columnCount; i++) { + String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { - case Types.ARRAY: - // no-op as of now - break; - case Types.BIGINT: - fields.add(new Field("long", FieldType.nullable(new ArrowType.Int(64, true)), null)); - break; - case Types.BINARY: case Types.LONGVARBINARY: case Types.VARBINARY: - break; - case Types.BIT: - break; - case Types.BLOB: - fields.add(new Field("binary", FieldType.nullable(new ArrowType.Binary()), null)); - break; case Types.BOOLEAN: + case Types.BIT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Bool()), null)); break; - case Types.CHAR: + case Types.TINYINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(8, true)), null)); break; - case Types.CLOB: + case Types.SMALLINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(16, true)), null)); break; - case Types.DATE: + case Types.INTEGER: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(32, true)), null)); break; - case Types.DECIMAL: case Types.NUMERIC: + case Types.BIGINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(64, true)), null)); break; - case Types.DOUBLE: case Types.FLOAT: - fields.add(new Field("double", FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); + case Types.NUMERIC: + case Types.DECIMAL: + int precision = rsmd.getPrecision(i); + int scale = rsmd.getScale(i); + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Decimal(precision, scale)), null)); break; - case Types.INTEGER: - fields.add(new Field("int", FieldType.nullable(new ArrowType.Int(32, true)), null)); + case Types.REAL: + case Types.FLOAT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); break; - case Types.LONGVARCHAR: case Types.LONGNVARCHAR: case Types.VARCHAR: case Types.NVARCHAR: + case Types.DOUBLE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(DOUBLE)), null)); break; - case Types.REAL: + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); break; - case Types.SMALLINT: + case Types.DATE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)); break; case Types.TIME: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); break; case Types.TIMESTAMP: + // timezone is null + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), null)); break; - case Types.TINYINT: + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.ARRAY: + // not handled +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: break; + case Types.BLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + default: -// throw new TypeConversionNotSupportedException(); // no-op - + break; } } return new Schema(fields.build(), null); } + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { + + assert rs != null; + assert root != null; + + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + + int rowCount = 0; + while (rs.next()) { + // for each column get the value based on the type + + for (int i = 1; i < columnCount; i++) { + String columnName = rsmd.getColumnName(i); + switch (rsmd.getColumnType(i)) { + case Types.BOOLEAN: + case Types.BIT: + BitVector bitVector = (BitVector) root.getVector(columnName); + bitVector.setInitialCapacity(1); + bitVector.allocateNew(); + bitVector.setSafe(rowCount,rs.getBoolean(i)? 1: 0); + bitVector.setValueCount(1); + break; + case Types.TINYINT: + TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); + tinyIntVector.setInitialCapacity(1); + tinyIntVector.allocateNew(); + tinyIntVector.setSafe(rowCount, rs.getInt(i)); + tinyIntVector.setValueCount(1); + break; + case Types.SMALLINT: + SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); + smallIntVector.setInitialCapacity(1); + smallIntVector.allocateNew(); + smallIntVector.setSafe(rowCount, rs.getInt(i)); + smallIntVector.setValueCount(1); + break; + case Types.INTEGER: + IntVector intVector = (IntVector)root.getVector(columnName); + intVector.setInitialCapacity(1); + intVector.allocateNew(); + intVector.setSafe(rowCount, rs.getInt(i)); + intVector.setValueCount(1); + break; + case Types.BIGINT: + BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); + bigIntVector.setInitialCapacity(1); + bigIntVector.allocateNew(); + bigIntVector.setSafe(rowCount, rs.getInt(i)); + bigIntVector.setValueCount(1); + break; + case Types.NUMERIC: + case Types.DECIMAL: + DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); + decimalVector.setInitialCapacity(1); + decimalVector.allocateNew(); + decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); + decimalVector.setValueCount(1); + break; + case Types.REAL: + case Types.FLOAT: + Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); + float4Vector.setInitialCapacity(1); + float4Vector.allocateNew(); + float4Vector.setSafe(rowCount, rs.getFloat(i)); + float4Vector.setValueCount(1); + break; + case Types.DOUBLE: + Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); + float8Vector.setInitialCapacity(1); + float8Vector.allocateNew(); + float8Vector.setSafe(rowCount, rs.getDouble(i)); + float8Vector.setValueCount(1); + break; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); + varcharVector.setInitialCapacity(1); + varcharVector.allocateNew(); + String value = rs.getString(i); + varcharVector.setIndexDefined(i); + varcharVector.setValueLengthSafe(i, value.length()); + varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); + varcharVector.setValueCount(1); + break; + case Types.DATE: + DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); + dateMilliVector.setInitialCapacity(1); + dateMilliVector.allocateNew(); + dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); + dateMilliVector.setValueCount(1); + break; + case Types.TIME: + TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); + timeMilliVector.setInitialCapacity(1); + timeMilliVector.allocateNew(); + timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? + timeMilliVector.setValueCount(1); + break; + case Types.TIMESTAMP: + // timezone is null + TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); + timeStampVector.setInitialCapacity(1); + timeStampVector.allocateNew(); + timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); + timeStampVector.setValueCount(1); + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: +// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.ARRAY: + // not handled +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: + break; + case Types.BLOB: +// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + + default: + // no-op + break; + } + rowCount++; + + } + } + } + } From e462f806e3a984f2fcd312595bdd71d9d2d1fe65 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 21 Feb 2018 06:08:45 +0530 Subject: [PATCH 003/129] Started adding required testcases related code. --- java/adapter/jdbc/pom.xml | 33 ++++- .../arrow/adapter/jdbc/JdbcToArrow.java | 5 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 31 ++++- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 80 +++++++++++ .../org/apache/arrow/adapter/jdbc/Table.java | 47 +++++++ .../{dbcp.properties => db.properties} | 4 +- .../jdbc/src/test/resources/test1_h2.yml | 129 ++++++++++++++++++ 7 files changed, 312 insertions(+), 17 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java rename java/adapter/jdbc/src/test/resources/{dbcp.properties => db.properties} (95%) create mode 100644 java/adapter/jdbc/src/test/resources/test1_h2.yml diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index cee23b0bc1f..8ccedc92bbe 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -21,12 +21,6 @@ Arrow JDBC Adapter http://maven.apache.org - - - org.apache.commons - commons-dbcp2 - 2.1 - @@ -40,6 +34,7 @@ arrow-vector 0.9.0-SNAPSHOT + junit @@ -47,6 +42,32 @@ 4.11 test + + + com.h2database + h2 + 1.4.196 + test + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.3.0 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.2.3 + test + + + org.apache.commons + commons-lang3 + 3.4 + test + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index cc9413a90e6..6a8c095f501 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -31,7 +31,7 @@ public class JdbcToArrow { /** * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. * - * @param connection Database connection to be used. This metho will not close hte passed connection object. Since hte caller has passed + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. * @return @@ -50,7 +50,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t VectorSchemaRoot root = VectorSchemaRoot.create( JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); JdbcToArrowUtils.jdbcToArrowVectors(rs, root); - + return root; } catch (Exception exc) { // just throw it out after logging throw exc; @@ -62,7 +62,6 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t stmt.close(); } } - return null; } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 32ad3064ee4..9b67218d5e8 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,10 +27,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Types; +import java.sql.*; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -255,16 +252,38 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: -// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; + varBinaryVector.setInitialCapacity(1); + varBinaryVector.allocateNew(); + byte[] bytes = rs.getBytes(i); + varBinaryVector.setIndexDefined(i); + varBinaryVector.setValueLengthSafe(i, bytes.length); + varBinaryVector.setSafe(i, bytes); break; case Types.ARRAY: // not handled // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); + varcharVector1.setInitialCapacity(1); + varcharVector1.allocateNew(); + Clob clob = rs.getClob(i); + int length = (int)clob.length(); + varcharVector1.setIndexDefined(i); + varcharVector1.setValueLengthSafe(i, length); + varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(0, length).getBytes(), 0, length); + varcharVector1.setValueCount(1); break; case Types.BLOB: -// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; + varBinaryVector1.setInitialCapacity(1); + varBinaryVector1.allocateNew(); + Blob blob = rs.getBlob(i); + byte[] data = blob.getBytes(0, (int)blob.length()); + varBinaryVector1.setIndexDefined(i); + varBinaryVector1.setValueLengthSafe(i, (int)blob.length()); + varBinaryVector1.setSafe(i, data); break; default: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java new file mode 100644 index 00000000000..d9165879e3c --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +/** + * + */ +public class JdbcToArrowTest { + + private Connection conn = null; + + @Before + public void setUp() throws Exception { + Properties properties = new Properties(); + properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + + Class.forName(properties.getProperty("driver")); + + conn = DriverManager + .getConnection(properties.getProperty("url"), properties);; + } + + @Test + public void sqlToArrowTest() throws Exception { + + // create the table and insert data for the test + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + try { +// Table table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + + + System.out.print(10); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + try { + + } finally { + + } + + } + + @After + public void destroy() throws Exception { + if (conn != null) { + conn.close(); + } + } + +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java new file mode 100644 index 00000000000..cbbb3a3fa2f --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +/** + * + */ +public class Table { + + private String create; + private String[] data; + + public Table() { + } + + public String getCreate() { + return create; + } + + public void setCreate(String create) { + this.create = create; + } + + public String[] getData() { + return data; + } + + public void setData(String[] data) { + this.data = data; + } +} diff --git a/java/adapter/jdbc/src/test/resources/dbcp.properties b/java/adapter/jdbc/src/test/resources/db.properties similarity index 95% rename from java/adapter/jdbc/src/test/resources/dbcp.properties rename to java/adapter/jdbc/src/test/resources/db.properties index aea0a132477..5973b652a88 100644 --- a/java/adapter/jdbc/src/test/resources/dbcp.properties +++ b/java/adapter/jdbc/src/test/resources/db.properties @@ -9,9 +9,9 @@ # OF ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -driverClassName=org.h2.Driver +driver=org.h2.Driver url=jdbc:h2:~/test -username=sa +user=sa password= diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml new file mode 100644 index 00000000000..da7c3fc74b8 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -0,0 +1,129 @@ + +create: CREATE TABLE table1 (int_field1 INT, bool_field2, BOOLEAN, tinyint_field3 TINYINT + title VARCHAR(50) NOT NULL, + author VARCHAR(20) NOT NULL, + submission_date DATE, + ); + +columns: +- name: int_field1 + type: INT +- name: bool_field2 + type: BOOLEAN +- name: tinyint_field3 + type: TINYINT +- name: smallint_field4 + type: SMALLINT +- name: bigint_field5 + type: BIGINT +- name: decimal_field6 + type: DECIMAL(20,2) +- name: double_field7 + type: DOUBLE +- name: real_field8 + type: REAL +- name: time_field9 + type: TIME +- name: date_field10 + type: DATE +- name: timestamp_field11 + type: TIMESTAMP +- name: binary_field12 + type: BINARY(100) +- name: varchar_field13 + type: VARCHAR(256) +- name: blob_field14 + type: BLOB +- name: clob_field15 + type: CLOB +- name: char_field16 + type: CHAR(16) +- name: bit_field17 + type: BIT + +data: +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' From 89d02bf13d6ae839afc2b7c047b45fd057bfa020 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 21 Feb 2018 06:09:43 +0530 Subject: [PATCH 004/129] Added *.yml under exclude list. --- java/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index 152deaa9c42..97c6dd07b24 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -132,6 +132,7 @@ **/*.linux **/client/build/** **/*.tbl + **/*.yml @@ -623,5 +624,6 @@ memory vector tools + adapter/jdbc From c9d27276c411e9906087a7cfe1b4c03ac4dea6ab Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 04:57:50 +0530 Subject: [PATCH 005/129] Added *.properties under exclude list. --- java/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/pom.xml b/java/pom.xml index 97c6dd07b24..363bea54936 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -133,6 +133,7 @@ **/client/build/** **/*.tbl **/*.yml + **/*.properties From b003abbeb8cf5a309e4c57eed8527e68e73e25a6 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 04:59:19 +0530 Subject: [PATCH 006/129] Added test case code - first cut. Used to YAML for the test data. Fixed issues in the vector creation for CLOB. --- java/adapter/jdbc/pom.xml | 24 +-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 3 +- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 60 +++++- .../org/apache/arrow/adapter/jdbc/Table.java | 18 ++ .../jdbc/src/test/resources/db.properties | 11 -- .../jdbc/src/test/resources/test1_h2.yml | 176 +++++------------- 6 files changed, 136 insertions(+), 156 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 8ccedc92bbe..cfb3d8dc4ad 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -55,18 +55,18 @@ 2.3.0 test - - com.fasterxml.jackson.core - jackson-databind - 2.2.3 - test - - - org.apache.commons - commons-lang3 - 3.4 - test - + + + + + + + + + + + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 9b67218d5e8..b551fb88431 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -127,6 +127,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); break; case Types.BLOB: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); @@ -272,7 +273,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int length = (int)clob.length(); varcharVector1.setIndexDefined(i); varcharVector1.setValueLengthSafe(i, length); - varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(0, length).getBytes(), 0, length); + varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); varcharVector1.setValueCount(1); break; case Types.BLOB: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index d9165879e3c..54040d7cc2b 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -27,6 +28,7 @@ import java.io.File; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.Statement; import java.util.Properties; /** @@ -35,37 +37,81 @@ public class JdbcToArrowTest { private Connection conn = null; + private Table table = null; @Before public void setUp() throws Exception { Properties properties = new Properties(); properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + Class.forName(properties.getProperty("driver")); conn = DriverManager .getConnection(properties.getProperty("url"), properties);; } - @Test - public void sqlToArrowTest() throws Exception { + private void createTestData() throws Exception { - // create the table and insert data for the test - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + Statement stmt = null; try { -// Table table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } + } + + } - System.out.print(10); + private void deleteTestData() throws Exception { + Statement stmt = null; + try { + stmt = conn.createStatement(); + stmt.executeUpdate(table.getDrop()); + } catch (Exception e) { - // TODO Auto-generated catch block e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } } + } + + @Test + public void sqlToArrowTest() throws Exception { + Statement stmt = null; try { + createTestData(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select * from " + table.getName() + ";"); + + System.out.print(root.getRowCount()); + + } catch (Exception e) { + e.printStackTrace(); } finally { + + if (stmt != null) { + stmt.close(); + } + + deleteTestData(); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index cbbb3a3fa2f..5ca1287bb88 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -23,12 +23,22 @@ */ public class Table { + private String name; private String create; private String[] data; + private String drop; public Table() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getCreate() { return create; } @@ -44,4 +54,12 @@ public String[] getData() { public void setData(String[] data) { this.data = data; } + + public String getDrop() { + return drop; + } + + public void setDrop(String drop) { + this.drop = drop; + } } diff --git a/java/adapter/jdbc/src/test/resources/db.properties b/java/adapter/jdbc/src/test/resources/db.properties index 5973b652a88..52c67cd72c7 100644 --- a/java/adapter/jdbc/src/test/resources/db.properties +++ b/java/adapter/jdbc/src/test/resources/db.properties @@ -1,14 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with this work for additional -# information regarding copyright ownership. The ASF licenses this file to -# You under the Apache License, Version 2.0 (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of -# the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required -# by applicable law or agreed to in writing, software distributed under the -# icense is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - driver=org.h2.Driver url=jdbc:h2:~/test user=sa diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml index da7c3fc74b8..5bdaf50b811 100644 --- a/java/adapter/jdbc/src/test/resources/test1_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -1,129 +1,55 @@ -create: CREATE TABLE table1 (int_field1 INT, bool_field2, BOOLEAN, tinyint_field3 TINYINT - title VARCHAR(50) NOT NULL, - author VARCHAR(20) NOT NULL, - submission_date DATE, - ); +name: 'table1' -columns: -- name: int_field1 - type: INT -- name: bool_field2 - type: BOOLEAN -- name: tinyint_field3 - type: TINYINT -- name: smallint_field4 - type: SMALLINT -- name: bigint_field5 - type: BIGINT -- name: decimal_field6 - type: DECIMAL(20,2) -- name: double_field7 - type: DOUBLE -- name: real_field8 - type: REAL -- name: time_field9 - type: TIME -- name: date_field10 - type: DATE -- name: timestamp_field11 - type: TIMESTAMP -- name: binary_field12 - type: BINARY(100) -- name: varchar_field13 - type: VARCHAR(256) -- name: blob_field14 - type: BLOB -- name: clob_field15 - type: CLOB -- name: char_field16 - type: CHAR(16) -- name: bit_field17 - type: BIT +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' data: -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +drop: 'DROP table table1;' \ No newline at end of file From 333bc017b2214a1cd89943d3e7b346b90e1c3cdb Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 06:33:07 +0530 Subject: [PATCH 007/129] Code changes to create poper vector objects. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 85 ++++++++++--------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index b551fb88431..25ac4759990 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -160,101 +160,101 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BOOLEAN: case Types.BIT: BitVector bitVector = (BitVector) root.getVector(columnName); - bitVector.setInitialCapacity(1); - bitVector.allocateNew(); - bitVector.setSafe(rowCount,rs.getBoolean(i)? 1: 0); - bitVector.setValueCount(1); + bitVector.setInitialCapacity(bitVector.getValueCapacity() + 1); + bitVector.reAlloc(); + bitVector.setSafe(rowCount, rs.getBoolean(i)? 1: 0); + bitVector.setValueCount(rowCount + 1); break; case Types.TINYINT: TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); - tinyIntVector.setInitialCapacity(1); - tinyIntVector.allocateNew(); + tinyIntVector.setInitialCapacity(tinyIntVector.getValueCapacity() + 1); + tinyIntVector.reAlloc(); tinyIntVector.setSafe(rowCount, rs.getInt(i)); - tinyIntVector.setValueCount(1); + tinyIntVector.setValueCount(rowCount + 1); break; case Types.SMALLINT: SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); - smallIntVector.setInitialCapacity(1); - smallIntVector.allocateNew(); + smallIntVector.setInitialCapacity(smallIntVector.getValueCapacity() + 1); + smallIntVector.reAlloc(); smallIntVector.setSafe(rowCount, rs.getInt(i)); - smallIntVector.setValueCount(1); + smallIntVector.setValueCount(rowCount + 1); break; case Types.INTEGER: IntVector intVector = (IntVector)root.getVector(columnName); - intVector.setInitialCapacity(1); - intVector.allocateNew(); + intVector.setInitialCapacity(intVector.getValueCapacity() + 1); + intVector.reAlloc(); intVector.setSafe(rowCount, rs.getInt(i)); - intVector.setValueCount(1); + intVector.setValueCount(rowCount + 1); break; case Types.BIGINT: BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); - bigIntVector.setInitialCapacity(1); - bigIntVector.allocateNew(); + bigIntVector.setInitialCapacity(bigIntVector.getValueCapacity() + 1); + bigIntVector.reAlloc(); bigIntVector.setSafe(rowCount, rs.getInt(i)); - bigIntVector.setValueCount(1); + bigIntVector.setValueCount(rowCount + 1); break; case Types.NUMERIC: case Types.DECIMAL: DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); - decimalVector.setInitialCapacity(1); - decimalVector.allocateNew(); + decimalVector.setInitialCapacity(decimalVector.getValueCapacity() + 1); + decimalVector.reAlloc(); decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); - decimalVector.setValueCount(1); + decimalVector.setValueCount(rowCount + 1); break; case Types.REAL: case Types.FLOAT: Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); - float4Vector.setInitialCapacity(1); - float4Vector.allocateNew(); + float4Vector.setInitialCapacity(float4Vector.getValueCapacity() + 1); + float4Vector.reAlloc(); float4Vector.setSafe(rowCount, rs.getFloat(i)); - float4Vector.setValueCount(1); + float4Vector.setValueCount(rowCount + 1); break; case Types.DOUBLE: Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); - float8Vector.setInitialCapacity(1); - float8Vector.allocateNew(); + float8Vector.setInitialCapacity(float8Vector.getValueCapacity() + 1); + float8Vector.reAlloc(); float8Vector.setSafe(rowCount, rs.getDouble(i)); - float8Vector.setValueCount(1); + float8Vector.setValueCount(rowCount + 1); break; case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - varcharVector.setInitialCapacity(1); + varcharVector.setInitialCapacity(varcharVector.getValueCapacity() + 1); varcharVector.allocateNew(); String value = rs.getString(i); varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); - varcharVector.setValueCount(1); + varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); - dateMilliVector.setInitialCapacity(1); - dateMilliVector.allocateNew(); + dateMilliVector.setInitialCapacity(dateMilliVector.getValueCapacity() + 1); + dateMilliVector.reAlloc(); dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); - dateMilliVector.setValueCount(1); + dateMilliVector.setValueCount(rowCount + 1); break; case Types.TIME: TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setInitialCapacity(1); - timeMilliVector.allocateNew(); + timeMilliVector.setInitialCapacity(timeMilliVector.getValueCapacity() + 1); + timeMilliVector.reAlloc(); timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? - timeMilliVector.setValueCount(1); + timeMilliVector.setValueCount(rowCount + 1); break; case Types.TIMESTAMP: // timezone is null TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setInitialCapacity(1); - timeStampVector.allocateNew(); + timeStampVector.setInitialCapacity(timeStampVector.getValueCapacity() + 1); + timeStampVector.reAlloc(); timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); - timeStampVector.setValueCount(1); + timeStampVector.setValueCount(rowCount + 1); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector.setInitialCapacity(1); + varBinaryVector.setInitialCapacity(varBinaryVector.getValueCapacity() + 1); varBinaryVector.allocateNew(); byte[] bytes = rs.getBytes(i); varBinaryVector.setIndexDefined(i); @@ -267,34 +267,35 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.CLOB: VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - varcharVector1.setInitialCapacity(1); + varcharVector1.setInitialCapacity(varcharVector1.getValueCapacity() + 1); varcharVector1.allocateNew(); Clob clob = rs.getClob(i); int length = (int)clob.length(); varcharVector1.setIndexDefined(i); varcharVector1.setValueLengthSafe(i, length); varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); - varcharVector1.setValueCount(1); + varcharVector1.setValueCount(rowCount + 1); break; case Types.BLOB: VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector1.setInitialCapacity(1); + varBinaryVector1.setInitialCapacity(varBinaryVector1.getValueCapacity() + 1); varBinaryVector1.allocateNew(); Blob blob = rs.getBlob(i); byte[] data = blob.getBytes(0, (int)blob.length()); varBinaryVector1.setIndexDefined(i); varBinaryVector1.setValueLengthSafe(i, (int)blob.length()); varBinaryVector1.setSafe(i, data); + varBinaryVector1.setValueCount(rowCount + 1); break; default: // no-op break; } - rowCount++; - } + rowCount++; } + root.setRowCount(rowCount); } From 32fa9686683245624ca4f44353bf8876df739aad Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 06:37:33 +0530 Subject: [PATCH 008/129] Code changes to create poper vector objects. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 25ac4759990..021225dbd6b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -154,6 +154,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw while (rs.next()) { // for each column get the value based on the type + // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i < columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { From 1e5c584641def0a1720267045fd8648db1274788 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 06:49:15 +0530 Subject: [PATCH 009/129] Code changes to allocate memory for the vectors before adding objects. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 6 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 60 +++++++------- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 2 +- .../jdbc/src/test/resources/test1_h2.yml | 80 ++++++++++--------- 4 files changed, 73 insertions(+), 75 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 6a8c095f501..2116b08a33a 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -34,10 +34,11 @@ public class JdbcToArrow { * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. + * @param size Size of the dataset to return * @return * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, int size) throws Exception { RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); @@ -49,7 +50,8 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t ResultSetMetaData rsmd = rs.getMetaData(); VectorSchemaRoot root = VectorSchemaRoot.create( JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); - JdbcToArrowUtils.jdbcToArrowVectors(rs, root); + JdbcToArrowUtils.allocateVectors(root, size); + JdbcToArrowUtils.jdbcToArrowVectors(rs, root, size); return root; } catch (Exception exc) { // just throw it out after logging diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 021225dbd6b..68c8b3dd87f 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,7 +27,12 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.math.BigDecimal; import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -142,7 +147,19 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti return new Schema(fields.build(), null); } - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { + public static void allocateVectors(VectorSchemaRoot root, int size) { + List vectors = root.getFieldVectors(); + for (FieldVector fieldVector: vectors) { + if (fieldVector instanceof BaseFixedWidthVector) { + ((BaseFixedWidthVector) fieldVector).allocateNew(size); + } else { + fieldVector.allocateNew(); + } + fieldVector.setInitialCapacity(size); + } + } + + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int size) throws Exception { assert rs != null; assert root != null; @@ -161,115 +178,91 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BOOLEAN: case Types.BIT: BitVector bitVector = (BitVector) root.getVector(columnName); - bitVector.setInitialCapacity(bitVector.getValueCapacity() + 1); - bitVector.reAlloc(); bitVector.setSafe(rowCount, rs.getBoolean(i)? 1: 0); bitVector.setValueCount(rowCount + 1); break; case Types.TINYINT: TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); - tinyIntVector.setInitialCapacity(tinyIntVector.getValueCapacity() + 1); - tinyIntVector.reAlloc(); tinyIntVector.setSafe(rowCount, rs.getInt(i)); tinyIntVector.setValueCount(rowCount + 1); break; case Types.SMALLINT: SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); - smallIntVector.setInitialCapacity(smallIntVector.getValueCapacity() + 1); - smallIntVector.reAlloc(); smallIntVector.setSafe(rowCount, rs.getInt(i)); smallIntVector.setValueCount(rowCount + 1); break; case Types.INTEGER: IntVector intVector = (IntVector)root.getVector(columnName); - intVector.setInitialCapacity(intVector.getValueCapacity() + 1); - intVector.reAlloc(); intVector.setSafe(rowCount, rs.getInt(i)); intVector.setValueCount(rowCount + 1); break; case Types.BIGINT: BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); - bigIntVector.setInitialCapacity(bigIntVector.getValueCapacity() + 1); - bigIntVector.reAlloc(); bigIntVector.setSafe(rowCount, rs.getInt(i)); bigIntVector.setValueCount(rowCount + 1); break; case Types.NUMERIC: case Types.DECIMAL: DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); - decimalVector.setInitialCapacity(decimalVector.getValueCapacity() + 1); - decimalVector.reAlloc(); decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); decimalVector.setValueCount(rowCount + 1); break; case Types.REAL: case Types.FLOAT: Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); - float4Vector.setInitialCapacity(float4Vector.getValueCapacity() + 1); - float4Vector.reAlloc(); float4Vector.setSafe(rowCount, rs.getFloat(i)); float4Vector.setValueCount(rowCount + 1); break; case Types.DOUBLE: Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); - float8Vector.setInitialCapacity(float8Vector.getValueCapacity() + 1); - float8Vector.reAlloc(); float8Vector.setSafe(rowCount, rs.getDouble(i)); float8Vector.setValueCount(rowCount + 1); break; case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - varcharVector.setInitialCapacity(varcharVector.getValueCapacity() + 1); - varcharVector.allocateNew(); String value = rs.getString(i); varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); - varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); + varcharVector.setSafe(rowCount, value.getBytes(), 0, value.length()); varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); - dateMilliVector.setInitialCapacity(dateMilliVector.getValueCapacity() + 1); - dateMilliVector.reAlloc(); dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); dateMilliVector.setValueCount(rowCount + 1); break; case Types.TIME: TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setInitialCapacity(timeMilliVector.getValueCapacity() + 1); - timeMilliVector.reAlloc(); timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? timeMilliVector.setValueCount(rowCount + 1); break; case Types.TIMESTAMP: // timezone is null TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setInitialCapacity(timeStampVector.getValueCapacity() + 1); - timeStampVector.reAlloc(); timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); timeStampVector.setValueCount(rowCount + 1); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector.setInitialCapacity(varBinaryVector.getValueCapacity() + 1); - varBinaryVector.allocateNew(); byte[] bytes = rs.getBytes(i); varBinaryVector.setIndexDefined(i); varBinaryVector.setValueLengthSafe(i, bytes.length); - varBinaryVector.setSafe(i, bytes); + varBinaryVector.setSafe(rowCount, bytes); + varBinaryVector.setValueCount(rowCount + 1); break; case Types.ARRAY: // not handled // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - varcharVector1.setInitialCapacity(varcharVector1.getValueCapacity() + 1); - varcharVector1.allocateNew(); Clob clob = rs.getClob(i); int length = (int)clob.length(); varcharVector1.setIndexDefined(i); @@ -278,9 +271,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw varcharVector1.setValueCount(rowCount + 1); break; case Types.BLOB: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector1.setInitialCapacity(varBinaryVector1.getValueCapacity() + 1); - varBinaryVector1.allocateNew(); Blob blob = rs.getBlob(i); byte[] data = blob.getBytes(0, (int)blob.length()); varBinaryVector1.setIndexDefined(i); @@ -300,4 +292,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw } + + } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 54040d7cc2b..fac0b4b22fb 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -98,7 +98,7 @@ public void sqlToArrowTest() throws Exception { try { createTestData(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select * from " + table.getName() + ";"); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select from " + table.getName() + ";", 5); System.out.print(root.getRowCount()); diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml index 5bdaf50b811..338a554383e 100644 --- a/java/adapter/jdbc/src/test/resources/test1_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -12,44 +12,46 @@ data: - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' drop: 'DROP table table1;' \ No newline at end of file From 127b4fb80718dae4f377b7ead96623888cdf981b Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 07:00:45 +0530 Subject: [PATCH 010/129] Fixed code to handle dataset size. Fixed code to handle only one column in select query. --- .../org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 5 ++++- .../org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 2 +- .../test/java/org/apache/arrow/adapter/jdbc/Table.java | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 68c8b3dd87f..7739ad0eb32 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -172,7 +172,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s // for each column get the value based on the type // need to change this to build Java lists and then build Arrow vectors - for (int i = 1; i < columnCount; i++) { + for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { case Types.BOOLEAN: @@ -287,6 +287,9 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s } } rowCount++; + if (rowCount == size) { + break; + } } root.setRowCount(rowCount); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index fac0b4b22fb..99c63dd3cc3 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -98,7 +98,7 @@ public void sqlToArrowTest() throws Exception { try { createTestData(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select from " + table.getName() + ";", 5); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select int_field1 from " + table.getName() + ";", 5); System.out.print(root.getRowCount()); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 5ca1287bb88..2d3a328de92 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -26,6 +26,7 @@ public class Table { private String name; private String create; private String[] data; + private String query; private String drop; public Table() { @@ -55,6 +56,14 @@ public void setData(String[] data) { this.data = data; } + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + public String getDrop() { return drop; } From 8e2b7f595a439f698f77a1309e609f412b85176a Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 07:20:20 +0530 Subject: [PATCH 011/129] Removed unused import. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 7739ad0eb32..231fbdd01f6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,12 +27,8 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.math.BigDecimal; import java.sql.*; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; From 772ae96c7f0c3c79e6455b7d4fb400778f1a31f9 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Sat, 24 Feb 2018 00:57:42 +0530 Subject: [PATCH 012/129] Test --- java/adapter/jdbc/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index cfb3d8dc4ad..853e528f60f 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -55,6 +55,7 @@ 2.3.0 test + From e3c490d4bea92c8b5420343b227ffdbb61f3d034 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 15 Feb 2018 06:22:46 +0530 Subject: [PATCH 013/129] Initial commit --- java/adapter/jdbc/pom.xml | 52 ++++++++ .../arrow/adapter/jdbc/JdbcToArrow.java | 68 ++++++++++ .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 122 ++++++++++++++++++ .../jdbc/src/test/resources/dbcp.properties | 18 +++ 4 files changed, 260 insertions(+) create mode 100644 java/adapter/jdbc/pom.xml create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java create mode 100644 java/adapter/jdbc/src/test/resources/dbcp.properties diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml new file mode 100644 index 00000000000..cee23b0bc1f --- /dev/null +++ b/java/adapter/jdbc/pom.xml @@ -0,0 +1,52 @@ + + + + + + 4.0.0 + org.apache.arrow.adapter.jdbc + arrow-jdbc + jar + 0.9.0-SNAPSHOT + Arrow JDBC Adapter + http://maven.apache.org + + + + org.apache.commons + commons-dbcp2 + 2.1 + + + + + org.apache.arrow + arrow-memory + 0.9.0-SNAPSHOT + + + + org.apache.arrow + arrow-vector + 0.9.0-SNAPSHOT + + + + junit + junit + 4.11 + test + + + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java new file mode 100644 index 00000000000..a60b59a7b9c --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.sql.*; + +/** + * + */ +public class JdbcToArrow { + + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * + * @param connection Database connection to be used. This metho will not close hte passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @return + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. + */ + public static VectorSchemaRoot convert(Connection connection, String query) throws SQLException { + + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + + Statement stmt = null; + ResultSet rs = null; + ResultSetMetaData rsmd = null; + try { + stmt = connection.createStatement(); + rs = stmt.executeQuery(query); + VectorSchemaRoot root = VectorSchemaRoot.create( + JdbcToArrowUtils.jdbcToArrowSchema(rs.getMetaData()), rootAllocator); + + } catch (SQLException exc) { + + } finally { + if (rs != null) { + rs.close(); + } + if (stmt != null) { + stmt.close(); + } + } + return null; + } + + + +} diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java new file mode 100644 index 00000000000..66fd5cef198 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import com.google.common.collect.ImmutableList; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; + +import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; + + +/** + * + */ +public class JdbcToArrowUtils { + + /** + * JDBC type Java type + CHAR String + VARCHAR String + LONGVARCHAR String + NUMERIC java.math.BigDecimal + DECIMAL java.math.BigDecimal + BIT boolean + TINYINT byte + SMALLINT short + INTEGER int + BIGINT long + REAL float + FLOAT double + DOUBLE double + BINARY byte[] + VARBINARY byte[] + LONGVARBINARY byte[] + DATE java.sql.Date + TIME java.sql.Time + TIMESTAMP java.sql.Timestamp + + * @param rsmd + * @return + * @throws SQLException + */ + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + ImmutableList.Builder fields = ImmutableList.builder(); + int columnCount = rsmd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + switch (rsmd.getColumnType(i)) { + case Types.ARRAY: + // no-op as of now + break; + case Types.BIGINT: + fields.add(new Field("long", FieldType.nullable(new ArrowType.Int(64, true)), null)); + break; + case Types.BINARY: case Types.LONGVARBINARY: case Types.VARBINARY: + break; + case Types.BIT: + break; + case Types.BLOB: + fields.add(new Field("binary", FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.BOOLEAN: + break; + case Types.CHAR: + break; + case Types.CLOB: + break; + case Types.DATE: + break; + case Types.DECIMAL: case Types.NUMERIC: + break; + case Types.DOUBLE: case Types.FLOAT: + fields.add(new Field("double", FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); + break; + case Types.INTEGER: + fields.add(new Field("int", FieldType.nullable(new ArrowType.Int(32, true)), null)); + break; + case Types.LONGVARCHAR: case Types.LONGNVARCHAR: case Types.VARCHAR: case Types.NVARCHAR: + break; + case Types.REAL: + break; + case Types.SMALLINT: + break; + case Types.TIME: + break; + case Types.TIMESTAMP: + break; + case Types.TINYINT: + break; + default: +// throw new TypeConversionNotSupportedException(); + // no-op + + } + } + + return new Schema(fields.build(), null); + } + + +} diff --git a/java/adapter/jdbc/src/test/resources/dbcp.properties b/java/adapter/jdbc/src/test/resources/dbcp.properties new file mode 100644 index 00000000000..aea0a132477 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/dbcp.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. The ASF licenses this file to +# You under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of +# the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +# by applicable law or agreed to in writing, software distributed under the +# icense is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +driverClassName=org.h2.Driver +url=jdbc:h2:~/test +username=sa +password= + + + From df444adc3814b272d665a0385f0866688eae6338 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 19 Feb 2018 02:40:56 +0530 Subject: [PATCH 014/129] Added necessary code for JDBC to Arrow schema creation and Arrow vector objects creation. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 12 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 213 +++++++++++++++--- 2 files changed, 193 insertions(+), 32 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index a60b59a7b9c..cc9413a90e6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -37,21 +37,23 @@ public class JdbcToArrow { * @return * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. */ - public static VectorSchemaRoot convert(Connection connection, String query) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); Statement stmt = null; ResultSet rs = null; - ResultSetMetaData rsmd = null; try { stmt = connection.createStatement(); rs = stmt.executeQuery(query); + ResultSetMetaData rsmd = rs.getMetaData(); VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(rs.getMetaData()), rootAllocator); - - } catch (SQLException exc) { + JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); + JdbcToArrowUtils.jdbcToArrowVectors(rs, root); + } catch (Exception exc) { + // just throw it out after logging + throw exc; } finally { if (rs != null) { rs.close(); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 66fd5cef198..32ad3064ee4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -19,15 +19,20 @@ package org.apache.arrow.adapter.jdbc; import com.google.common.collect.ImmutableList; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -63,60 +68,214 @@ public class JdbcToArrowUtils { * @throws SQLException */ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + + assert rsmd != null; + ImmutableList.Builder fields = ImmutableList.builder(); int columnCount = rsmd.getColumnCount(); for (int i = 1; i <= columnCount; i++) { + String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { - case Types.ARRAY: - // no-op as of now - break; - case Types.BIGINT: - fields.add(new Field("long", FieldType.nullable(new ArrowType.Int(64, true)), null)); - break; - case Types.BINARY: case Types.LONGVARBINARY: case Types.VARBINARY: - break; - case Types.BIT: - break; - case Types.BLOB: - fields.add(new Field("binary", FieldType.nullable(new ArrowType.Binary()), null)); - break; case Types.BOOLEAN: + case Types.BIT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Bool()), null)); break; - case Types.CHAR: + case Types.TINYINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(8, true)), null)); break; - case Types.CLOB: + case Types.SMALLINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(16, true)), null)); break; - case Types.DATE: + case Types.INTEGER: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(32, true)), null)); break; - case Types.DECIMAL: case Types.NUMERIC: + case Types.BIGINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(64, true)), null)); break; - case Types.DOUBLE: case Types.FLOAT: - fields.add(new Field("double", FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); + case Types.NUMERIC: + case Types.DECIMAL: + int precision = rsmd.getPrecision(i); + int scale = rsmd.getScale(i); + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Decimal(precision, scale)), null)); break; - case Types.INTEGER: - fields.add(new Field("int", FieldType.nullable(new ArrowType.Int(32, true)), null)); + case Types.REAL: + case Types.FLOAT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); break; - case Types.LONGVARCHAR: case Types.LONGNVARCHAR: case Types.VARCHAR: case Types.NVARCHAR: + case Types.DOUBLE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(DOUBLE)), null)); break; - case Types.REAL: + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); break; - case Types.SMALLINT: + case Types.DATE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)); break; case Types.TIME: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); break; case Types.TIMESTAMP: + // timezone is null + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), null)); break; - case Types.TINYINT: + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.ARRAY: + // not handled +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: break; + case Types.BLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + default: -// throw new TypeConversionNotSupportedException(); // no-op - + break; } } return new Schema(fields.build(), null); } + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { + + assert rs != null; + assert root != null; + + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + + int rowCount = 0; + while (rs.next()) { + // for each column get the value based on the type + + for (int i = 1; i < columnCount; i++) { + String columnName = rsmd.getColumnName(i); + switch (rsmd.getColumnType(i)) { + case Types.BOOLEAN: + case Types.BIT: + BitVector bitVector = (BitVector) root.getVector(columnName); + bitVector.setInitialCapacity(1); + bitVector.allocateNew(); + bitVector.setSafe(rowCount,rs.getBoolean(i)? 1: 0); + bitVector.setValueCount(1); + break; + case Types.TINYINT: + TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); + tinyIntVector.setInitialCapacity(1); + tinyIntVector.allocateNew(); + tinyIntVector.setSafe(rowCount, rs.getInt(i)); + tinyIntVector.setValueCount(1); + break; + case Types.SMALLINT: + SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); + smallIntVector.setInitialCapacity(1); + smallIntVector.allocateNew(); + smallIntVector.setSafe(rowCount, rs.getInt(i)); + smallIntVector.setValueCount(1); + break; + case Types.INTEGER: + IntVector intVector = (IntVector)root.getVector(columnName); + intVector.setInitialCapacity(1); + intVector.allocateNew(); + intVector.setSafe(rowCount, rs.getInt(i)); + intVector.setValueCount(1); + break; + case Types.BIGINT: + BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); + bigIntVector.setInitialCapacity(1); + bigIntVector.allocateNew(); + bigIntVector.setSafe(rowCount, rs.getInt(i)); + bigIntVector.setValueCount(1); + break; + case Types.NUMERIC: + case Types.DECIMAL: + DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); + decimalVector.setInitialCapacity(1); + decimalVector.allocateNew(); + decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); + decimalVector.setValueCount(1); + break; + case Types.REAL: + case Types.FLOAT: + Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); + float4Vector.setInitialCapacity(1); + float4Vector.allocateNew(); + float4Vector.setSafe(rowCount, rs.getFloat(i)); + float4Vector.setValueCount(1); + break; + case Types.DOUBLE: + Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); + float8Vector.setInitialCapacity(1); + float8Vector.allocateNew(); + float8Vector.setSafe(rowCount, rs.getDouble(i)); + float8Vector.setValueCount(1); + break; + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); + varcharVector.setInitialCapacity(1); + varcharVector.allocateNew(); + String value = rs.getString(i); + varcharVector.setIndexDefined(i); + varcharVector.setValueLengthSafe(i, value.length()); + varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); + varcharVector.setValueCount(1); + break; + case Types.DATE: + DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); + dateMilliVector.setInitialCapacity(1); + dateMilliVector.allocateNew(); + dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); + dateMilliVector.setValueCount(1); + break; + case Types.TIME: + TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); + timeMilliVector.setInitialCapacity(1); + timeMilliVector.allocateNew(); + timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? + timeMilliVector.setValueCount(1); + break; + case Types.TIMESTAMP: + // timezone is null + TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); + timeStampVector.setInitialCapacity(1); + timeStampVector.allocateNew(); + timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); + timeStampVector.setValueCount(1); + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: +// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.ARRAY: + // not handled +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: + break; + case Types.BLOB: +// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + + default: + // no-op + break; + } + rowCount++; + + } + } + } + } From 15f281ff0c9a6ce90eaf438c56d399dee8d18691 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 21 Feb 2018 06:08:45 +0530 Subject: [PATCH 015/129] Started adding required testcases related code. --- java/adapter/jdbc/pom.xml | 33 ++++- .../arrow/adapter/jdbc/JdbcToArrow.java | 5 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 31 ++++- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 80 +++++++++++ .../org/apache/arrow/adapter/jdbc/Table.java | 47 +++++++ .../{dbcp.properties => db.properties} | 4 +- .../jdbc/src/test/resources/test1_h2.yml | 129 ++++++++++++++++++ 7 files changed, 312 insertions(+), 17 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java rename java/adapter/jdbc/src/test/resources/{dbcp.properties => db.properties} (95%) create mode 100644 java/adapter/jdbc/src/test/resources/test1_h2.yml diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index cee23b0bc1f..8ccedc92bbe 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -21,12 +21,6 @@ Arrow JDBC Adapter http://maven.apache.org - - - org.apache.commons - commons-dbcp2 - 2.1 - @@ -40,6 +34,7 @@ arrow-vector 0.9.0-SNAPSHOT + junit @@ -47,6 +42,32 @@ 4.11 test + + + com.h2database + h2 + 1.4.196 + test + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.3.0 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.2.3 + test + + + org.apache.commons + commons-lang3 + 3.4 + test + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index cc9413a90e6..6a8c095f501 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -31,7 +31,7 @@ public class JdbcToArrow { /** * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. * - * @param connection Database connection to be used. This metho will not close hte passed connection object. Since hte caller has passed + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. * @return @@ -50,7 +50,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t VectorSchemaRoot root = VectorSchemaRoot.create( JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); JdbcToArrowUtils.jdbcToArrowVectors(rs, root); - + return root; } catch (Exception exc) { // just throw it out after logging throw exc; @@ -62,7 +62,6 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t stmt.close(); } } - return null; } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 32ad3064ee4..9b67218d5e8 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,10 +27,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Types; +import java.sql.*; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -255,16 +252,38 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: -// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; + varBinaryVector.setInitialCapacity(1); + varBinaryVector.allocateNew(); + byte[] bytes = rs.getBytes(i); + varBinaryVector.setIndexDefined(i); + varBinaryVector.setValueLengthSafe(i, bytes.length); + varBinaryVector.setSafe(i, bytes); break; case Types.ARRAY: // not handled // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); + varcharVector1.setInitialCapacity(1); + varcharVector1.allocateNew(); + Clob clob = rs.getClob(i); + int length = (int)clob.length(); + varcharVector1.setIndexDefined(i); + varcharVector1.setValueLengthSafe(i, length); + varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(0, length).getBytes(), 0, length); + varcharVector1.setValueCount(1); break; case Types.BLOB: -// fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; + varBinaryVector1.setInitialCapacity(1); + varBinaryVector1.allocateNew(); + Blob blob = rs.getBlob(i); + byte[] data = blob.getBytes(0, (int)blob.length()); + varBinaryVector1.setIndexDefined(i); + varBinaryVector1.setValueLengthSafe(i, (int)blob.length()); + varBinaryVector1.setSafe(i, data); break; default: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java new file mode 100644 index 00000000000..d9165879e3c --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +/** + * + */ +public class JdbcToArrowTest { + + private Connection conn = null; + + @Before + public void setUp() throws Exception { + Properties properties = new Properties(); + properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + + Class.forName(properties.getProperty("driver")); + + conn = DriverManager + .getConnection(properties.getProperty("url"), properties);; + } + + @Test + public void sqlToArrowTest() throws Exception { + + // create the table and insert data for the test + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + try { +// Table table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + + + System.out.print(10); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + try { + + } finally { + + } + + } + + @After + public void destroy() throws Exception { + if (conn != null) { + conn.close(); + } + } + +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java new file mode 100644 index 00000000000..cbbb3a3fa2f --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +/** + * + */ +public class Table { + + private String create; + private String[] data; + + public Table() { + } + + public String getCreate() { + return create; + } + + public void setCreate(String create) { + this.create = create; + } + + public String[] getData() { + return data; + } + + public void setData(String[] data) { + this.data = data; + } +} diff --git a/java/adapter/jdbc/src/test/resources/dbcp.properties b/java/adapter/jdbc/src/test/resources/db.properties similarity index 95% rename from java/adapter/jdbc/src/test/resources/dbcp.properties rename to java/adapter/jdbc/src/test/resources/db.properties index aea0a132477..5973b652a88 100644 --- a/java/adapter/jdbc/src/test/resources/dbcp.properties +++ b/java/adapter/jdbc/src/test/resources/db.properties @@ -9,9 +9,9 @@ # OF ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -driverClassName=org.h2.Driver +driver=org.h2.Driver url=jdbc:h2:~/test -username=sa +user=sa password= diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml new file mode 100644 index 00000000000..da7c3fc74b8 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -0,0 +1,129 @@ + +create: CREATE TABLE table1 (int_field1 INT, bool_field2, BOOLEAN, tinyint_field3 TINYINT + title VARCHAR(50) NOT NULL, + author VARCHAR(20) NOT NULL, + submission_date DATE, + ); + +columns: +- name: int_field1 + type: INT +- name: bool_field2 + type: BOOLEAN +- name: tinyint_field3 + type: TINYINT +- name: smallint_field4 + type: SMALLINT +- name: bigint_field5 + type: BIGINT +- name: decimal_field6 + type: DECIMAL(20,2) +- name: double_field7 + type: DOUBLE +- name: real_field8 + type: REAL +- name: time_field9 + type: TIME +- name: date_field10 + type: DATE +- name: timestamp_field11 + type: TIMESTAMP +- name: binary_field12 + type: BINARY(100) +- name: varchar_field13 + type: VARCHAR(256) +- name: blob_field14 + type: BLOB +- name: clob_field15 + type: CLOB +- name: char_field16 + type: CHAR(16) +- name: bit_field17 + type: BIT + +data: +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' +- - '101' + - '1' + - '45' + - '12000' + - '9223372036854775807' + - '17345667789.23' + - '56478356785.345' + - '56478356785.345' + - '12:45:35' + - '2018-02-12' + - '2018-02-12 12:45:35' + - 'some text that needs to be converted to binary' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to blob' + - 'some text that needs to be converted to clob' + - 'some char text' + - '0' From a32f788ed2848974d0a099a34fcc0073195487be Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 21 Feb 2018 06:09:43 +0530 Subject: [PATCH 016/129] Added *.yml under exclude list. --- java/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index 152deaa9c42..97c6dd07b24 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -132,6 +132,7 @@ **/*.linux **/client/build/** **/*.tbl + **/*.yml @@ -623,5 +624,6 @@ memory vector tools + adapter/jdbc From 1b4175ffcfd5960ed05f04885f95046f0e8da81d Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 04:57:50 +0530 Subject: [PATCH 017/129] Added *.properties under exclude list. --- java/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/pom.xml b/java/pom.xml index 97c6dd07b24..363bea54936 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -133,6 +133,7 @@ **/client/build/** **/*.tbl **/*.yml + **/*.properties From 4c706f54af08d98436bea8ab59b5c4ab48724604 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 04:59:19 +0530 Subject: [PATCH 018/129] Added test case code - first cut. Used to YAML for the test data. Fixed issues in the vector creation for CLOB. --- java/adapter/jdbc/pom.xml | 24 +-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 3 +- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 60 +++++- .../org/apache/arrow/adapter/jdbc/Table.java | 18 ++ .../jdbc/src/test/resources/db.properties | 11 -- .../jdbc/src/test/resources/test1_h2.yml | 176 +++++------------- 6 files changed, 136 insertions(+), 156 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 8ccedc92bbe..cfb3d8dc4ad 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -55,18 +55,18 @@ 2.3.0 test - - com.fasterxml.jackson.core - jackson-databind - 2.2.3 - test - - - org.apache.commons - commons-lang3 - 3.4 - test - + + + + + + + + + + + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 9b67218d5e8..b551fb88431 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -127,6 +127,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); break; case Types.BLOB: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); @@ -272,7 +273,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int length = (int)clob.length(); varcharVector1.setIndexDefined(i); varcharVector1.setValueLengthSafe(i, length); - varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(0, length).getBytes(), 0, length); + varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); varcharVector1.setValueCount(1); break; case Types.BLOB: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index d9165879e3c..54040d7cc2b 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -27,6 +28,7 @@ import java.io.File; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.Statement; import java.util.Properties; /** @@ -35,37 +37,81 @@ public class JdbcToArrowTest { private Connection conn = null; + private Table table = null; @Before public void setUp() throws Exception { Properties properties = new Properties(); properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + Class.forName(properties.getProperty("driver")); conn = DriverManager .getConnection(properties.getProperty("url"), properties);; } - @Test - public void sqlToArrowTest() throws Exception { + private void createTestData() throws Exception { - // create the table and insert data for the test - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + Statement stmt = null; try { -// Table table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } + } + + } - System.out.print(10); + private void deleteTestData() throws Exception { + Statement stmt = null; + try { + stmt = conn.createStatement(); + stmt.executeUpdate(table.getDrop()); + } catch (Exception e) { - // TODO Auto-generated catch block e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } } + } + + @Test + public void sqlToArrowTest() throws Exception { + Statement stmt = null; try { + createTestData(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select * from " + table.getName() + ";"); + + System.out.print(root.getRowCount()); + + } catch (Exception e) { + e.printStackTrace(); } finally { + + if (stmt != null) { + stmt.close(); + } + + deleteTestData(); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index cbbb3a3fa2f..5ca1287bb88 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -23,12 +23,22 @@ */ public class Table { + private String name; private String create; private String[] data; + private String drop; public Table() { } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getCreate() { return create; } @@ -44,4 +54,12 @@ public String[] getData() { public void setData(String[] data) { this.data = data; } + + public String getDrop() { + return drop; + } + + public void setDrop(String drop) { + this.drop = drop; + } } diff --git a/java/adapter/jdbc/src/test/resources/db.properties b/java/adapter/jdbc/src/test/resources/db.properties index 5973b652a88..52c67cd72c7 100644 --- a/java/adapter/jdbc/src/test/resources/db.properties +++ b/java/adapter/jdbc/src/test/resources/db.properties @@ -1,14 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more contributor -# license agreements. See the NOTICE file distributed with this work for additional -# information regarding copyright ownership. The ASF licenses this file to -# You under the Apache License, Version 2.0 (the "License"); you may not use -# this file except in compliance with the License. You may obtain a copy of -# the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required -# by applicable law or agreed to in writing, software distributed under the -# icense is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -# OF ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - driver=org.h2.Driver url=jdbc:h2:~/test user=sa diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml index da7c3fc74b8..5bdaf50b811 100644 --- a/java/adapter/jdbc/src/test/resources/test1_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -1,129 +1,55 @@ -create: CREATE TABLE table1 (int_field1 INT, bool_field2, BOOLEAN, tinyint_field3 TINYINT - title VARCHAR(50) NOT NULL, - author VARCHAR(20) NOT NULL, - submission_date DATE, - ); +name: 'table1' -columns: -- name: int_field1 - type: INT -- name: bool_field2 - type: BOOLEAN -- name: tinyint_field3 - type: TINYINT -- name: smallint_field4 - type: SMALLINT -- name: bigint_field5 - type: BIGINT -- name: decimal_field6 - type: DECIMAL(20,2) -- name: double_field7 - type: DOUBLE -- name: real_field8 - type: REAL -- name: time_field9 - type: TIME -- name: date_field10 - type: DATE -- name: timestamp_field11 - type: TIMESTAMP -- name: binary_field12 - type: BINARY(100) -- name: varchar_field13 - type: VARCHAR(256) -- name: blob_field14 - type: BLOB -- name: clob_field15 - type: CLOB -- name: char_field16 - type: CHAR(16) -- name: bit_field17 - type: BIT +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' data: -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' -- - '101' - - '1' - - '45' - - '12000' - - '9223372036854775807' - - '17345667789.23' - - '56478356785.345' - - '56478356785.345' - - '12:45:35' - - '2018-02-12' - - '2018-02-12 12:45:35' - - 'some text that needs to be converted to binary' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to blob' - - 'some text that needs to be converted to clob' - - 'some char text' - - '0' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +drop: 'DROP table table1;' \ No newline at end of file From 9ead27de00d2f7c6dbee5c768fe6b8ef9ea0cb8f Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 06:33:07 +0530 Subject: [PATCH 019/129] Code changes to create poper vector objects. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 85 ++++++++++--------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index b551fb88431..25ac4759990 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -160,101 +160,101 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BOOLEAN: case Types.BIT: BitVector bitVector = (BitVector) root.getVector(columnName); - bitVector.setInitialCapacity(1); - bitVector.allocateNew(); - bitVector.setSafe(rowCount,rs.getBoolean(i)? 1: 0); - bitVector.setValueCount(1); + bitVector.setInitialCapacity(bitVector.getValueCapacity() + 1); + bitVector.reAlloc(); + bitVector.setSafe(rowCount, rs.getBoolean(i)? 1: 0); + bitVector.setValueCount(rowCount + 1); break; case Types.TINYINT: TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); - tinyIntVector.setInitialCapacity(1); - tinyIntVector.allocateNew(); + tinyIntVector.setInitialCapacity(tinyIntVector.getValueCapacity() + 1); + tinyIntVector.reAlloc(); tinyIntVector.setSafe(rowCount, rs.getInt(i)); - tinyIntVector.setValueCount(1); + tinyIntVector.setValueCount(rowCount + 1); break; case Types.SMALLINT: SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); - smallIntVector.setInitialCapacity(1); - smallIntVector.allocateNew(); + smallIntVector.setInitialCapacity(smallIntVector.getValueCapacity() + 1); + smallIntVector.reAlloc(); smallIntVector.setSafe(rowCount, rs.getInt(i)); - smallIntVector.setValueCount(1); + smallIntVector.setValueCount(rowCount + 1); break; case Types.INTEGER: IntVector intVector = (IntVector)root.getVector(columnName); - intVector.setInitialCapacity(1); - intVector.allocateNew(); + intVector.setInitialCapacity(intVector.getValueCapacity() + 1); + intVector.reAlloc(); intVector.setSafe(rowCount, rs.getInt(i)); - intVector.setValueCount(1); + intVector.setValueCount(rowCount + 1); break; case Types.BIGINT: BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); - bigIntVector.setInitialCapacity(1); - bigIntVector.allocateNew(); + bigIntVector.setInitialCapacity(bigIntVector.getValueCapacity() + 1); + bigIntVector.reAlloc(); bigIntVector.setSafe(rowCount, rs.getInt(i)); - bigIntVector.setValueCount(1); + bigIntVector.setValueCount(rowCount + 1); break; case Types.NUMERIC: case Types.DECIMAL: DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); - decimalVector.setInitialCapacity(1); - decimalVector.allocateNew(); + decimalVector.setInitialCapacity(decimalVector.getValueCapacity() + 1); + decimalVector.reAlloc(); decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); - decimalVector.setValueCount(1); + decimalVector.setValueCount(rowCount + 1); break; case Types.REAL: case Types.FLOAT: Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); - float4Vector.setInitialCapacity(1); - float4Vector.allocateNew(); + float4Vector.setInitialCapacity(float4Vector.getValueCapacity() + 1); + float4Vector.reAlloc(); float4Vector.setSafe(rowCount, rs.getFloat(i)); - float4Vector.setValueCount(1); + float4Vector.setValueCount(rowCount + 1); break; case Types.DOUBLE: Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); - float8Vector.setInitialCapacity(1); - float8Vector.allocateNew(); + float8Vector.setInitialCapacity(float8Vector.getValueCapacity() + 1); + float8Vector.reAlloc(); float8Vector.setSafe(rowCount, rs.getDouble(i)); - float8Vector.setValueCount(1); + float8Vector.setValueCount(rowCount + 1); break; case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - varcharVector.setInitialCapacity(1); + varcharVector.setInitialCapacity(varcharVector.getValueCapacity() + 1); varcharVector.allocateNew(); String value = rs.getString(i); varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); - varcharVector.setValueCount(1); + varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); - dateMilliVector.setInitialCapacity(1); - dateMilliVector.allocateNew(); + dateMilliVector.setInitialCapacity(dateMilliVector.getValueCapacity() + 1); + dateMilliVector.reAlloc(); dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); - dateMilliVector.setValueCount(1); + dateMilliVector.setValueCount(rowCount + 1); break; case Types.TIME: TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setInitialCapacity(1); - timeMilliVector.allocateNew(); + timeMilliVector.setInitialCapacity(timeMilliVector.getValueCapacity() + 1); + timeMilliVector.reAlloc(); timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? - timeMilliVector.setValueCount(1); + timeMilliVector.setValueCount(rowCount + 1); break; case Types.TIMESTAMP: // timezone is null TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setInitialCapacity(1); - timeStampVector.allocateNew(); + timeStampVector.setInitialCapacity(timeStampVector.getValueCapacity() + 1); + timeStampVector.reAlloc(); timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); - timeStampVector.setValueCount(1); + timeStampVector.setValueCount(rowCount + 1); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector.setInitialCapacity(1); + varBinaryVector.setInitialCapacity(varBinaryVector.getValueCapacity() + 1); varBinaryVector.allocateNew(); byte[] bytes = rs.getBytes(i); varBinaryVector.setIndexDefined(i); @@ -267,34 +267,35 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.CLOB: VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - varcharVector1.setInitialCapacity(1); + varcharVector1.setInitialCapacity(varcharVector1.getValueCapacity() + 1); varcharVector1.allocateNew(); Clob clob = rs.getClob(i); int length = (int)clob.length(); varcharVector1.setIndexDefined(i); varcharVector1.setValueLengthSafe(i, length); varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); - varcharVector1.setValueCount(1); + varcharVector1.setValueCount(rowCount + 1); break; case Types.BLOB: VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector1.setInitialCapacity(1); + varBinaryVector1.setInitialCapacity(varBinaryVector1.getValueCapacity() + 1); varBinaryVector1.allocateNew(); Blob blob = rs.getBlob(i); byte[] data = blob.getBytes(0, (int)blob.length()); varBinaryVector1.setIndexDefined(i); varBinaryVector1.setValueLengthSafe(i, (int)blob.length()); varBinaryVector1.setSafe(i, data); + varBinaryVector1.setValueCount(rowCount + 1); break; default: // no-op break; } - rowCount++; - } + rowCount++; } + root.setRowCount(rowCount); } From 73b019840c1a5549aafa960608bd2e9c7e584dc2 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 22 Feb 2018 06:37:33 +0530 Subject: [PATCH 020/129] Code changes to create poper vector objects. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 25ac4759990..021225dbd6b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -154,6 +154,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw while (rs.next()) { // for each column get the value based on the type + // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i < columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { From c97e910c84670e2e146ab07e374a8e740b21facd Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 06:49:15 +0530 Subject: [PATCH 021/129] Code changes to allocate memory for the vectors before adding objects. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 6 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 60 +++++++------- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 2 +- .../jdbc/src/test/resources/test1_h2.yml | 80 ++++++++++--------- 4 files changed, 73 insertions(+), 75 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 6a8c095f501..2116b08a33a 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -34,10 +34,11 @@ public class JdbcToArrow { * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. + * @param size Size of the dataset to return * @return * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, int size) throws Exception { RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); @@ -49,7 +50,8 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t ResultSetMetaData rsmd = rs.getMetaData(); VectorSchemaRoot root = VectorSchemaRoot.create( JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); - JdbcToArrowUtils.jdbcToArrowVectors(rs, root); + JdbcToArrowUtils.allocateVectors(root, size); + JdbcToArrowUtils.jdbcToArrowVectors(rs, root, size); return root; } catch (Exception exc) { // just throw it out after logging diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 021225dbd6b..68c8b3dd87f 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,7 +27,12 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.math.BigDecimal; import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -142,7 +147,19 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti return new Schema(fields.build(), null); } - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { + public static void allocateVectors(VectorSchemaRoot root, int size) { + List vectors = root.getFieldVectors(); + for (FieldVector fieldVector: vectors) { + if (fieldVector instanceof BaseFixedWidthVector) { + ((BaseFixedWidthVector) fieldVector).allocateNew(size); + } else { + fieldVector.allocateNew(); + } + fieldVector.setInitialCapacity(size); + } + } + + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int size) throws Exception { assert rs != null; assert root != null; @@ -161,115 +178,91 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BOOLEAN: case Types.BIT: BitVector bitVector = (BitVector) root.getVector(columnName); - bitVector.setInitialCapacity(bitVector.getValueCapacity() + 1); - bitVector.reAlloc(); bitVector.setSafe(rowCount, rs.getBoolean(i)? 1: 0); bitVector.setValueCount(rowCount + 1); break; case Types.TINYINT: TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); - tinyIntVector.setInitialCapacity(tinyIntVector.getValueCapacity() + 1); - tinyIntVector.reAlloc(); tinyIntVector.setSafe(rowCount, rs.getInt(i)); tinyIntVector.setValueCount(rowCount + 1); break; case Types.SMALLINT: SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); - smallIntVector.setInitialCapacity(smallIntVector.getValueCapacity() + 1); - smallIntVector.reAlloc(); smallIntVector.setSafe(rowCount, rs.getInt(i)); smallIntVector.setValueCount(rowCount + 1); break; case Types.INTEGER: IntVector intVector = (IntVector)root.getVector(columnName); - intVector.setInitialCapacity(intVector.getValueCapacity() + 1); - intVector.reAlloc(); intVector.setSafe(rowCount, rs.getInt(i)); intVector.setValueCount(rowCount + 1); break; case Types.BIGINT: BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); - bigIntVector.setInitialCapacity(bigIntVector.getValueCapacity() + 1); - bigIntVector.reAlloc(); bigIntVector.setSafe(rowCount, rs.getInt(i)); bigIntVector.setValueCount(rowCount + 1); break; case Types.NUMERIC: case Types.DECIMAL: DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); - decimalVector.setInitialCapacity(decimalVector.getValueCapacity() + 1); - decimalVector.reAlloc(); decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); decimalVector.setValueCount(rowCount + 1); break; case Types.REAL: case Types.FLOAT: Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); - float4Vector.setInitialCapacity(float4Vector.getValueCapacity() + 1); - float4Vector.reAlloc(); float4Vector.setSafe(rowCount, rs.getFloat(i)); float4Vector.setValueCount(rowCount + 1); break; case Types.DOUBLE: Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); - float8Vector.setInitialCapacity(float8Vector.getValueCapacity() + 1); - float8Vector.reAlloc(); float8Vector.setSafe(rowCount, rs.getDouble(i)); float8Vector.setValueCount(rowCount + 1); break; case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - varcharVector.setInitialCapacity(varcharVector.getValueCapacity() + 1); - varcharVector.allocateNew(); String value = rs.getString(i); varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); - varcharVector.setSafe(varcharVector.getValueCapacity(), value.getBytes(), 0, value.length()); + varcharVector.setSafe(rowCount, value.getBytes(), 0, value.length()); varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); - dateMilliVector.setInitialCapacity(dateMilliVector.getValueCapacity() + 1); - dateMilliVector.reAlloc(); dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); dateMilliVector.setValueCount(rowCount + 1); break; case Types.TIME: TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setInitialCapacity(timeMilliVector.getValueCapacity() + 1); - timeMilliVector.reAlloc(); timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? timeMilliVector.setValueCount(rowCount + 1); break; case Types.TIMESTAMP: // timezone is null TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setInitialCapacity(timeStampVector.getValueCapacity() + 1); - timeStampVector.reAlloc(); timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); timeStampVector.setValueCount(rowCount + 1); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector.setInitialCapacity(varBinaryVector.getValueCapacity() + 1); - varBinaryVector.allocateNew(); byte[] bytes = rs.getBytes(i); varBinaryVector.setIndexDefined(i); varBinaryVector.setValueLengthSafe(i, bytes.length); - varBinaryVector.setSafe(i, bytes); + varBinaryVector.setSafe(rowCount, bytes); + varBinaryVector.setValueCount(rowCount + 1); break; case Types.ARRAY: // not handled // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - varcharVector1.setInitialCapacity(varcharVector1.getValueCapacity() + 1); - varcharVector1.allocateNew(); Clob clob = rs.getClob(i); int length = (int)clob.length(); varcharVector1.setIndexDefined(i); @@ -278,9 +271,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw varcharVector1.setValueCount(rowCount + 1); break; case Types.BLOB: + // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector1.setInitialCapacity(varBinaryVector1.getValueCapacity() + 1); - varBinaryVector1.allocateNew(); Blob blob = rs.getBlob(i); byte[] data = blob.getBytes(0, (int)blob.length()); varBinaryVector1.setIndexDefined(i); @@ -300,4 +292,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw } + + } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 54040d7cc2b..fac0b4b22fb 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -98,7 +98,7 @@ public void sqlToArrowTest() throws Exception { try { createTestData(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select * from " + table.getName() + ";"); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select from " + table.getName() + ";", 5); System.out.print(root.getRowCount()); diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_h2.yml index 5bdaf50b811..338a554383e 100644 --- a/java/adapter/jdbc/src/test/resources/test1_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_h2.yml @@ -12,44 +12,46 @@ data: - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' drop: 'DROP table table1;' \ No newline at end of file From 945922127652b96320318fc8851b9b82cdb1134d Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 07:00:45 +0530 Subject: [PATCH 022/129] Fixed code to handle dataset size. Fixed code to handle only one column in select query. --- .../org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 5 ++++- .../org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 2 +- .../test/java/org/apache/arrow/adapter/jdbc/Table.java | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 68c8b3dd87f..7739ad0eb32 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -172,7 +172,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s // for each column get the value based on the type // need to change this to build Java lists and then build Arrow vectors - for (int i = 1; i < columnCount; i++) { + for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { case Types.BOOLEAN: @@ -287,6 +287,9 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s } } rowCount++; + if (rowCount == size) { + break; + } } root.setRowCount(rowCount); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index fac0b4b22fb..99c63dd3cc3 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -98,7 +98,7 @@ public void sqlToArrowTest() throws Exception { try { createTestData(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select from " + table.getName() + ";", 5); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select int_field1 from " + table.getName() + ";", 5); System.out.print(root.getRowCount()); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 5ca1287bb88..2d3a328de92 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -26,6 +26,7 @@ public class Table { private String name; private String create; private String[] data; + private String query; private String drop; public Table() { @@ -55,6 +56,14 @@ public void setData(String[] data) { this.data = data; } + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + public String getDrop() { return drop; } From 5c1f5f216c8b6f05f65bf3cfb2d9db77185a6ba3 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Feb 2018 07:20:20 +0530 Subject: [PATCH 023/129] Removed unused import. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 7739ad0eb32..231fbdd01f6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,12 +27,8 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.math.BigDecimal; import java.sql.*; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; From f76ac486b87db4ff8b66ffc8cd4e74b6fce2f268 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Sat, 24 Feb 2018 00:57:42 +0530 Subject: [PATCH 024/129] Test --- java/adapter/jdbc/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index cfb3d8dc4ad..853e528f60f 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -55,6 +55,7 @@ 2.3.0 test + From a4d2b32d2aeeee78463a1185617436ac862c1963 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Sat, 24 Feb 2018 05:05:29 +0530 Subject: [PATCH 025/129] Removed the size parameter from the API. Added new test file --- .../arrow/adapter/jdbc/JdbcToArrow.java | 6 ++-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 17 +++++------ .../arrow/adapter/jdbc/JdbcToArrowTest.java | 27 ++++++++---------- .../jdbc/src/test/resources/test2_h2.yml | 28 +++++++++++++++++++ 4 files changed, 49 insertions(+), 29 deletions(-) create mode 100644 java/adapter/jdbc/src/test/resources/test2_h2.yml diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 2116b08a33a..6a8c095f501 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -34,11 +34,10 @@ public class JdbcToArrow { * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. - * @param size Size of the dataset to return * @return * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, int size) throws Exception { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); @@ -50,8 +49,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, i ResultSetMetaData rsmd = rs.getMetaData(); VectorSchemaRoot root = VectorSchemaRoot.create( JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); - JdbcToArrowUtils.allocateVectors(root, size); - JdbcToArrowUtils.jdbcToArrowVectors(rs, root, size); + JdbcToArrowUtils.jdbcToArrowVectors(rs, root); return root; } catch (Exception exc) { // just throw it out after logging diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 231fbdd01f6..cecc006ac25 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -39,6 +39,8 @@ */ public class JdbcToArrowUtils { + private static final int DEFAULT_BUFFER_SIZE = 256; + /** * JDBC type Java type CHAR String @@ -143,7 +145,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti return new Schema(fields.build(), null); } - public static void allocateVectors(VectorSchemaRoot root, int size) { + private static void allocateVectors(VectorSchemaRoot root, int size) { List vectors = root.getFieldVectors(); for (FieldVector fieldVector: vectors) { if (fieldVector instanceof BaseFixedWidthVector) { @@ -155,7 +157,7 @@ public static void allocateVectors(VectorSchemaRoot root, int size) { } } - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int size) throws Exception { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { assert rs != null; assert root != null; @@ -163,6 +165,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s ResultSetMetaData rsmd = rs.getMetaData(); int columnCount = rsmd.getColumnCount(); + allocateVectors(root, DEFAULT_BUFFER_SIZE); + int rowCount = 0; while (rs.next()) { // for each column get the value based on the type @@ -217,7 +221,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: - // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); String value = rs.getString(i); varcharVector.setIndexDefined(i); @@ -232,7 +235,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s break; case Types.TIME: TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); // TODO - down conversion cast?? + timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); timeMilliVector.setValueCount(rowCount + 1); break; case Types.TIMESTAMP: @@ -244,7 +247,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: - // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; byte[] bytes = rs.getBytes(i); varBinaryVector.setIndexDefined(i); @@ -257,7 +259,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: - // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); Clob clob = rs.getClob(i); int length = (int)clob.length(); @@ -267,7 +268,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s varcharVector1.setValueCount(rowCount + 1); break; case Types.BLOB: - // TODO - How to handle the buffer size if it starts exceeding the iniital allocated buffer VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; Blob blob = rs.getBlob(i); byte[] data = blob.getBytes(0, (int)blob.length()); @@ -283,9 +283,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, int s } } rowCount++; - if (rowCount == size) { - break; - } } root.setRowCount(rowCount); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 99c63dd3cc3..4dabb181363 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -37,15 +37,14 @@ public class JdbcToArrowTest { private Connection conn = null; - private Table table = null; + private ObjectMapper mapper = null; @Before public void setUp() throws Exception { Properties properties = new Properties(); properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - table = mapper.readValue(new File("/home/datum/codebase/arrow/java/adapter/jdbc/src/test/resources/test1_h2.yml"), Table.class); + mapper = new ObjectMapper(new YAMLFactory()); Class.forName(properties.getProperty("driver")); @@ -53,7 +52,7 @@ public void setUp() throws Exception { .getConnection(properties.getProperty("url"), properties);; } - private void createTestData() throws Exception { + private void createTestData(Table table) throws Exception { Statement stmt = null; try { @@ -76,7 +75,7 @@ private void createTestData() throws Exception { } - private void deleteTestData() throws Exception { + private void deleteTestData(Table table) throws Exception { Statement stmt = null; try { stmt = conn.createStatement(); @@ -94,24 +93,22 @@ private void deleteTestData() throws Exception { @Test public void sqlToArrowTest() throws Exception { - Statement stmt = null; + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test2_h2.yml"), + Table.class); + try { - createTestData(); + createTestData(table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, "select int_field1 from " + table.getName() + ";", 5); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); System.out.print(root.getRowCount()); } catch (Exception e) { e.printStackTrace(); } finally { - - - if (stmt != null) { - stmt.close(); - } - - deleteTestData(); + deleteTestData(table); } } diff --git a/java/adapter/jdbc/src/test/resources/test2_h2.yml b/java/adapter/jdbc/src/test/resources/test2_h2.yml new file mode 100644 index 00000000000..2f124f83d69 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test2_h2.yml @@ -0,0 +1,28 @@ + +name: 'table2' + +create: 'CREATE TABLE table2 (int_field1 INT);' + +data: + - 'INSERT INTO table2 VALUES (101);' + - 'INSERT INTO table2 VALUES (102);' + - 'INSERT INTO table2 VALUES (103);' + - 'INSERT INTO table2 VALUES (104);' + - 'INSERT INTO table2 VALUES (105);' + - 'INSERT INTO table2 VALUES (106);' + - 'INSERT INTO table2 VALUES (107);' + - 'INSERT INTO table2 VALUES (108);' + - 'INSERT INTO table2 VALUES (109);' + - 'INSERT INTO table2 VALUES (110);' + - 'INSERT INTO table2 VALUES (111);' + - 'INSERT INTO table2 VALUES (112);' + - 'INSERT INTO table2 VALUES (113);' + - 'INSERT INTO table2 VALUES (114);' + - 'INSERT INTO table2 VALUES (115);' + - 'INSERT INTO table2 VALUES (116);' + - 'INSERT INTO table2 VALUES (117);' + - 'INSERT INTO table2 VALUES (118);' + +query: 'select int_field1 from table2;' + +drop: 'DROP table table2;' \ No newline at end of file From 7f70a67424d82dad12c3c8d22e814dae219213db Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Sat, 24 Feb 2018 06:24:19 +0530 Subject: [PATCH 026/129] Added doc comment. --- .../main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 6a8c095f501..9399fb6b217 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -24,7 +24,7 @@ import java.sql.*; /** - * + * Utility class to convert JDBC objects to columnar Arrow format objects. */ public class JdbcToArrow { From c2ac474e0d7c72fcf663c000849fe56a4b59abed Mon Sep 17 00:00:00 2001 From: yashpal thakur Date: Tue, 27 Feb 2018 13:01:20 +0530 Subject: [PATCH 027/129] Committed JdbcToArrowTest.java with a new method to test commit and push in forked branch --- .../org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 4dabb181363..59de6aeeb4b 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -119,5 +119,14 @@ public void destroy() throws Exception { conn.close(); } } + + /** + * This Method returns named resource as input stream from classpath + * @param name of the resource + * @return resourec as InputStream + */ + private InputStream getResource(String name) { + return this.getClass().getClassLoader().getResourceAsStream(name); + } } From f834c7771862f64b2300023fdc2d6d876f7a625b Mon Sep 17 00:00:00 2001 From: yashpal thakur Date: Tue, 27 Feb 2018 16:07:11 +0530 Subject: [PATCH 028/129] added missing import statement --- .../test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 59de6aeeb4b..9ecc1c5636c 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -26,6 +26,7 @@ import org.junit.Test; import java.io.File; +import java.io.InputStream; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; From fdb9d711e6e36dd1bc079eb014c665a03f0b8c5b Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 14:45:23 -0700 Subject: [PATCH 029/129] Added null checks for the JDBC to Arrow conversion. Added new test cases. Added test helper class. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 66 ++++-- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 34 ++- .../adapter/jdbc/JdbcToArrowTestHelper.java | 220 ++++++++++++++++++ .../jdbc/src/test/resources/test3_h2.yml | 96 ++++++++ 4 files changed, 387 insertions(+), 29 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java create mode 100644 java/adapter/jdbc/src/test/resources/test3_h2.yml diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index cecc006ac25..38c3ec501f3 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -222,37 +222,57 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.VARCHAR: case Types.LONGVARCHAR: VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - String value = rs.getString(i); + String value = rs.getString(i) != null ? rs.getString(i) : ""; varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); varcharVector.setSafe(rowCount, value.getBytes(), 0, value.length()); varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: - DateMilliVector dateMilliVector = (DateMilliVector)root.getVector(columnName); - dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); + Date date = rs.getDate(i); + DateMilliVector dateMilliVector = (DateMilliVector) root.getVector(columnName); dateMilliVector.setValueCount(rowCount + 1); + if (date != null) { + dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); + } else { + dateMilliVector.setNull(rowCount); + } break; case Types.TIME: + Time time = rs.getTime(i); TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setSafe(rowCount, (int)rs.getTime(i).getTime()); timeMilliVector.setValueCount(rowCount + 1); + if (time != null) { + timeMilliVector.setSafe(rowCount, (int) rs.getTime(i).getTime()); + } else { + timeMilliVector.setNull(rowCount); + } + break; case Types.TIMESTAMP: // timezone is null + Timestamp timestamp = rs.getTimestamp(i); TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setSafe(rowCount, rs.getTimestamp(i).getTime()); timeStampVector.setValueCount(rowCount + 1); + if (timestamp != null) { + timeStampVector.setSafe(rowCount, timestamp.getTime()); + } else { + timeStampVector.setNull(rowCount); + } break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - byte[] bytes = rs.getBytes(i); - varBinaryVector.setIndexDefined(i); - varBinaryVector.setValueLengthSafe(i, bytes.length); - varBinaryVector.setSafe(rowCount, bytes); varBinaryVector.setValueCount(rowCount + 1); + byte[] bytes = rs.getBytes(i); + if (bytes != null) { + varBinaryVector.setIndexDefined(i); + varBinaryVector.setValueLengthSafe(i, bytes.length); + varBinaryVector.setSafe(rowCount, bytes); + } else { + varBinaryVector.setNull(rowCount); + } break; case Types.ARRAY: // not handled @@ -260,21 +280,29 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.CLOB: VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - Clob clob = rs.getClob(i); - int length = (int)clob.length(); - varcharVector1.setIndexDefined(i); - varcharVector1.setValueLengthSafe(i, length); - varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); varcharVector1.setValueCount(rowCount + 1); + Clob clob = rs.getClob(i); + if (clob != null) { + int length = (int) clob.length(); + varcharVector1.setIndexDefined(i); + varcharVector1.setValueLengthSafe(i, length); + varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); + } else { + varcharVector1.setNull(rowCount); + } break; case Types.BLOB: VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - Blob blob = rs.getBlob(i); - byte[] data = blob.getBytes(0, (int)blob.length()); - varBinaryVector1.setIndexDefined(i); - varBinaryVector1.setValueLengthSafe(i, (int)blob.length()); - varBinaryVector1.setSafe(i, data); varBinaryVector1.setValueCount(rowCount + 1); + Blob blob = rs.getBlob(i); + if (blob != null) { + byte[] data = blob.getBytes(0, (int) blob.length()); + varBinaryVector1.setIndexDefined(i); + varBinaryVector1.setValueLengthSafe(i, (int) blob.length()); + varBinaryVector1.setSafe(i, data); + } else { + varBinaryVector1.setNull(rowCount);} + break; default: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 9ecc1c5636c..59dc6eb2da9 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -92,7 +92,30 @@ private void deleteTestData(Table table) throws Exception { } @Test - public void sqlToArrowTest() throws Exception { + public void sqlToArrowTest1() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_h2.yml"), + Table.class); + + try { + createTestData(table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + + System.out.print(root.getRowCount()); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void sqlToArrowTest2() throws Exception { Table table = mapper.readValue( @@ -120,14 +143,5 @@ public void destroy() throws Exception { conn.close(); } } - - /** - * This Method returns named resource as input stream from classpath - * @param name of the resource - * @return resourec as InputStream - */ - private InputStream getResource(String name) { - return this.getClass().getClassLoader().getResourceAsStream(name); - } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java new file mode 100644 index 00000000000..181a05e626b --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.adapter.jdbc; + +import java.math.BigDecimal; + +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; + + +/** + * This is a Helper class which has functionalities to read and print the values from FieldVector object + * + */ +public class JdbcToArrowTestHelper { + + public void getIntVectorValues(FieldVector fx){ + IntVector intVector = ((IntVector) fx); + for(int j = 0; j < intVector.getValueCount(); j++){ + if(!intVector.isNull(j)){ + int value = intVector.get(j); + System.out.println("Int Vector Value [" + j +"] : " + value); + } else { + System.out.println("Int Vector Value [" + j +"] : NULL "); + } + } + } + + public void getBigIntVectorValues(FieldVector fx){ + BigIntVector bigIntVector = ((BigIntVector)fx); + for(int j = 0; j < bigIntVector.getValueCount(); j++){ + if(!bigIntVector.isNull(j)){ + long value = bigIntVector.get(j); + System.out.println("Big Int Vector Value [" + j +"] : " + value); + } else { + System.out.println("Big Int Vector Value [" + j +"] : NULL "); + } + } + } + + public void getVarBinaryVectorValues(FieldVector fx){ + VarBinaryVector varBinaryVector =((VarBinaryVector) fx); + for(int j = 0; j < varBinaryVector.getValueCount(); j++){ + if(!varBinaryVector.isNull(j)){ + byte[] value = varBinaryVector.get(j); + long valHash = hashArray(value); + System.out.println("Var Binary Vector Value [" + j +"] : " + firstX(value, 5)); + } else { + System.out.println("Var Binary Vector Value [" + j +"] : NULL "); + } + } + } + + public void getFloat4VectorValues(FieldVector fx){ + Float4Vector float4Vector = ((Float4Vector)fx); + for(int j = 0; j < float4Vector.getValueCount(); j++){ + if(!float4Vector.isNull(j)){ + float value = float4Vector.get(j); + System.out.println("Float Vector Value [" + j +"] : " + value); + } else { + System.out.println("Float Vector Value [" + j +"] : NULL "); + } + } + } + + public void getFloat8VectorValues(FieldVector fx){ + Float8Vector float8Vector = ((Float8Vector)fx); + for(int j = 0; j < float8Vector.getValueCount(); j++){ + if(!float8Vector.isNull(j)){ + double value = float8Vector.get(j); + System.out.println("Double Vector Value [" + j +"] : " + value); + } else { + System.out.println("Double Vector Value [" + j +"] : NULL "); + } + } + } + + public void getBitBooleanVectorValues(FieldVector fx){ + BitVector bitVector = ((BitVector)fx); + for(int j = 0; j < bitVector.getValueCount(); j++){ + if(!bitVector.isNull(j)){ + int value = bitVector.get(j); + System.out.println("Bit Boolean Vector Value[" + j +"] : " + value); + } else { + System.out.println("Bit Boolean Vector Value[" + j +"] : NULL "); + } + } + } + + public void getTinyIntVectorValues(FieldVector fx){ + TinyIntVector tinyIntVector = ((TinyIntVector)fx); + for(int j = 0; j < tinyIntVector.getValueCount(); j++){ + if(!tinyIntVector.isNull(j)){ + byte value = tinyIntVector.get(j); + System.out.println("Tiny Int Vector Value[" + j +"] : " + value); + } else { + System.out.println("Tiny Int Vector Value[" + j +"] : NULL "); + } + } + } + + public void getSmallIntVectorValues(FieldVector fx){ + SmallIntVector smallIntVector = ((SmallIntVector)fx); + for(int j = 0; j < smallIntVector.getValueCount(); j++){ + if(!smallIntVector.isNull(j)){ + short value = smallIntVector.get(j); + System.out.println("Small Int Vector Value[" + j +"] : " + value); + } else { + System.out.println("Small Int Vector Value[" + j +"] : NULL "); + } + } + } + + public void getDecimalVectorValues(FieldVector fx){ + DecimalVector decimalVector = ((DecimalVector)fx); + for(int j = 0; j < decimalVector.getValueCount(); j++){ + if(!decimalVector.isNull(j)){ + BigDecimal value = decimalVector.getObject(j); + System.out.println("Decimal Vector Value[" + j +"] : " + value); + } else { + System.out.println("Decimal Vector Value[" + j +"] : NULL "); + } + } + } + + public void getDateVectorValues(FieldVector fx){ + DateMilliVector dateMilliVector = ((DateMilliVector)fx); + for(int j = 0; j < dateMilliVector.getValueCount(); j++){ + if(!dateMilliVector.isNull(j)){ + long value = dateMilliVector.get(j); + System.out.println("Date Milli Vector Value[" + j +"] : " + value); + } else { + System.out.println("Date Milli Vector Value[" + j +"] : NULL "); + } + } + } + + public void getTimeVectorValues(FieldVector fx){ + TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); + for(int j = 0; j < timeMilliVector.getValueCount(); j++){ + if(!timeMilliVector.isNull(j)){ + int value = timeMilliVector.get(j); + System.out.println("Time Milli Vector Value[" + j +"] : " + value); + } else { + System.out.println("Time Milli Vector Value[" + j +"] : NULL "); + } + } + } + + public void getTimeStampVectorValues(FieldVector fx){ + TimeStampVector timeStampVector = ((TimeStampVector)fx); + for(int j = 0; j < timeStampVector.getValueCount(); j++){ + if(!timeStampVector.isNull(j)){ + long value = timeStampVector.get(j); + System.out.println("Time Stamp Vector Value [" + j +"] : " + value); + } else { + System.out.println("Time Stamp Vector Value [" + j +"] : NULL "); + } + } + } + + public void getVarcharVectorValues(FieldVector fx){ + VarCharVector varCharVector = ((VarCharVector)fx); + for(int j = 0; j < varCharVector.getValueCount(); j++){ + if(!varCharVector.isNull(j)){ + byte[] valArr = varCharVector.get(j); + //Text value = varCharVector.getObject(j); + //System.out.println("Var Char Vector Value [" + j +"] : " + value.toString()); + System.out.println("Var Char Accessor as byte[" + j +"] " + firstX(valArr, 5)); + } else { + System.out.println("\t\t varCharAccessor[" + j +"] : NULL "); + } + } + } + + public static long hashArray(byte[] data){ + long ret = 0; + for(int i = 0; i < data.length;i++) + ret+=data[i]; + return ret; + } + + public static String firstX(byte[] data, int items){ + int toProcess = Math.min(items, data.length); + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < toProcess; i++) { + sb.append(String.format("0x%02x", data[i])+ " "); + } + return sb.toString(); + } + +} diff --git a/java/adapter/jdbc/src/test/resources/test3_h2.yml b/java/adapter/jdbc/src/test/resources/test3_h2.yml new file mode 100644 index 00000000000..845b5ba717f --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test3_h2.yml @@ -0,0 +1,96 @@ +#--- Created TABLE -1 : +CREATE TABLE H2Table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT, identity_field18 IDENTITY, varchar_ignorecase_field19 VARCHAR_IGNORECASE(30), UUID_field20 UUID, float_field21 FLOAT); + +#--- Inserted values in TABLE -1 : +INSERT INTO H2Table1 VALUES (101, 1, 41, 12001, 92233721, 17345667781.23, 56478356781.345, 56478356781.345, '1:45:35', '2018-02-1', '2018-02-1 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as varchar1', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as clob1', 'some char text1', 1, 10001, 'varchar_ignorecase_1', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); +INSERT INTO H2Table1 VALUES (102, 0, 42, 12002, 92233722, 17345667782.23, 56478356782.345, 56478356782.345, '2:45:35', '2018-02-2', '2018-02-2 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as varchar2', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as clob2', 'some char text2', 0, 10002, 'varchar_ignorecase_2', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); +INSERT INTO H2Table1 VALUES (103, 1, 43, 12003, 92233723, 17345667783.23, 56478356783.345, 56478356783.345, '3:45:35', '2018-02-3', '2018-02-3 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as varchar3', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as clob3', 'some char text3', 1, 10003, 'varchar_ignorecase_3', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13'); +INSERT INTO H2Table1 VALUES (104, 0, 44, 12004, 92233724, 17345667784.23, 56478356784.345, 56478356784.345, '4:45:35', '2018-02-4', '2018-02-4 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some as varchar4', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some text as clob4', 'some char text4', 0, 10004, 'varchar_ignorecase_4', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14'); +INSERT INTO H2Table1 VALUES (105, 1, 45, 12005, 92233725, 17345667785.23, 56478356785.345, 56478356785.345, '5:45:35', '2018-02-5', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as varchar5', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob5', 'some char text5', 1, 10005, 'varchar_ignorecase_5', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15'); + +#--- Created TABLE -2 : +CREATE TABLE H2Table2 (int_T2field1 INT, bool_T2field2 BOOLEAN, tinyint_T2field3 TINYINT, smallint_T2field4 SMALLINT, bigint_T2field5 BIGINT, decimal_T2field6 DECIMAL(20,2), double_T2field7 DOUBLE, real_T2field8 REAL, time_T2field9 TIME, date_T2field10 DATE, timestamp_T2field11 TIMESTAMP, binary_T2field12 BINARY(100), varchar_T2field13 VARCHAR(256), blob_T2field14 BLOB, clob_T2field15 CLOB, char_T2field16 CHAR(16), bit_T2field17 BIT, identity_T2field18 IDENTITY, varchar_ignorecase_T2field19 VARCHAR_IGNORECASE(30), UUID_T2field20 UUID, float_T2field21 FLOAT); + +#--- Inserted values in TABLE -2 : +INSERT INTO H2Table2 VALUES (106, 1, 46, 12006, 92233726, 17345667786.23, 56478356786.345, 56478356786.345, '6:45:35', '2018-02-6', '2018-02-6 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as varchar6', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as clob6', 'some char text6', 0, 10006, 'varchar_ignorecase_6', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16'); +INSERT INTO H2Table2 VALUES (107, 0, 47, 12007, 92233727, 17345667787.23, 56478356787.345, 56478356787.345, '7:45:35', '2018-02-7','2018-02-7 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as varchar7', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as clob7', 'some char text7', 1, 10007, 'varchar_ignorecase_7', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17'); +INSERT INTO H2Table2 VALUES (108, 1, 48, 02008, 92233728, 17345667788.23, 56478356788.345, 56478356788.345, '8:45:35', '2018-02-8', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as varchar8', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as clob8', 'some char text8', 1, 10008, 'varchar_ignorecase_8', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18'); +INSERT INTO H2Table2 VALUES (109, 0, 49, 12009, 92233729, 17345667789.23, 56478356789.345, 56478356789.345, '9:45:35', '2018-02-9', '2018-02-9 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as varchar9', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as clob9', 'some char text9', 0, 10009, 'varchar_ignorecase_9', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a19'); +INSERT INTO H2Table2 VALUES (110, 1, 50, 12010, 92233710, 17345667710.23, 56478356710.345, 56478356710.345, '10:45:35', '2018-02-10', '2018-02-10 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617210', 'some text as varchar10', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob10', 'some char text10', 1, 10010, 'varchar_ignorecase_10', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a10'); + +#--- Created TABLE -3 : +CREATE TABLE H2Table4 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); + +--- Inserted values in TABLE -3 : +INSERT INTO H2Table4 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); +INSERT INTO H2Table4 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); +INSERT INTO H2Table4 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); +INSERT INTO H2Table4 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); +INSERT INTO H2Table4 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); +INSERT INTO H2Table4 VALUES (214, 92233214, 17345667214.23, 56478356214.345, 'some text as varchar214'); + +#--- Created TABLE -4 : +CREATE TABLE H2Table5 (int_T5field1 INT, bigint_T5field2 BIGINT, decimal_T5field3 DECIMAL(20,2), float_T5field4 FLOAT, varchar_T5field5 VARCHAR(256)); + +#--- Inserted values in TABLE -4 : +INSERT INTO H2Table5 VALUES (206, 92233206, 17345667206.23, 56478356206.345, 'some text as varchar206'); +INSERT INTO H2Table5 VALUES (207, 92233207, 17345667207.23, 56478356207.345, 'some text as varchar207'); +INSERT INTO H2Table5 VALUES (208, 92233208, 17345667208.23, 56478356208.345, 'some text as varchar208'); +INSERT INTO H2Table5 VALUES (209, 92233209, 17345667209.23, 56478356209.345, 'some text as varchar209'); +INSERT INTO H2Table5 VALUES (210, 92233210, 17345667210.23, 56478356210.345, 'some text as varchar210'); + +#--- Created TABLE -5 : +CREATE TABLE H2Table6 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); + +#--- Inserted values in TABLE -5 : +INSERT INTO H2Table6 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); +INSERT INTO H2Table6 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); +INSERT INTO H2Table6 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); +INSERT INTO H2Table6 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); +INSERT INTO H2Table6 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); +INSERT INTO H2Table6 VALUES (212, 92233212, 17345667212.23, 56478356212.345, 'some text as varchar212'); + +#----------------------------------------------------------------------------------------------------------------------------------- + + +#---- JOINS QUERIES +SELECT * from H2Table4 AS T4 INNER JOIN H2Table6 AS T6 ON T4.int_T4field1 = T6.int_T4field1; +SELECT * from H2Table4 AS T4 RIGHT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; +SELECT * from H2Table4 AS T4 LEFT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; +SELECT * from H2Table4 CROSS JOIN H2Table6; +SELECT * from H2Table6 CROSS JOIN H2Table5; + +#---- SET/UNION OPERATIONS QUERIES +SELECT int_T4field1 FROM H2Table4 UNION SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 UNION ALL SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 INTERSECT SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 MINUS SELECT int_T4field1 FROM H2Table6; + +#---- ORDER BY CLAUSE QUERIES +SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 ASC; +SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 DESC; + +#---- GROUP BY CLAUSE QUERIES +SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2; + +#---- HAVING CLAUSE QUERIES +SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2 +HAVING h2t1.bool_field2 IS TRUE AND h2t1.tinyint_field3 > 41; + +#---- CASE STATEMENT QUERY +SELECT int_field1, BOOL_FIELD2, tinyint_field3, CASE WHEN tinyint_field3 > 44 THEN TRUE ELSE FALSE END FROM H2Table1; + +#---- AGGREGATE FUNCTIONS QUERIES +SELECT MAX(int_field1) AS Int_Val, MAX(tinyint_field3) AS TinyInt_Val, MAX(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT MIN(int_field1) AS Int_Val, MIN(tinyint_field3) AS TinyInt_Val, MIN(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT AVG(int_field1) AS Int_Val, AVG(tinyint_field3) AS TinyInt_Val, AVG(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT SUM(int_field1) AS Int_Val, SUM(tinyint_field3) AS TinyInt_Val, SUM(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT COUNT(int_field1) AS Int_Val, COUNT(tinyint_field3) AS TinyInt_Val, COUNT(smallint_field4) AS SmallInt_Val FROM H2Table1; + +SELECT char_field16 FROM H2Table1; +SELECT UCASE (char_field16) AS Char_Val, UCASE (varchar_field13) AS VarChar_Val FROM H2Table1; +SELECT LCASE (char_field16) AS Char_Val, LCASE (varchar_field13) AS VarChar_Val FROM H2Table1; + +SELECT ROUND (decimal_field6, 2) AS Decimal_Val, ROUND (float_field21, 1) Float_Val, +ROUND (double_field7, 3) AS Double_Val, ROUND (real_field8, 4) AS Real_Val FROM H2Table1; + From ff4c79b88d7cd20590100187101f1ee6bdbfabb9 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 14:52:38 -0700 Subject: [PATCH 030/129] Test --- .../main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 9399fb6b217..2799cf67678 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -59,7 +59,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t rs.close(); } if (stmt != null) { - stmt.close(); + stmt.close(); // test } } } From 646eba63ff1d0fb5312ff045f71fa65d7212d0f9 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 14:58:34 -0700 Subject: [PATCH 031/129] Changed the build version to 0.10.0-SNAPSHOT --- java/adapter/jdbc/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 853e528f60f..8249a95f2ec 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -17,7 +17,7 @@ org.apache.arrow.adapter.jdbc arrow-jdbc jar - 0.9.0-SNAPSHOT + 0.10.0-SNAPSHOT Arrow JDBC Adapter http://maven.apache.org From 8c2190252e4db9db3fb87a5ba44f0c6f909ced31 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 15:44:48 -0700 Subject: [PATCH 032/129] Added necessary assertions. --- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 10 ++++++++-- .../adapter/jdbc/JdbcToArrowTestHelper.java | 17 ++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 59dc6eb2da9..2a5fa25dc9a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -104,7 +104,10 @@ public void sqlToArrowTest1() throws Exception { VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); - System.out.print(root.getRowCount()); + int[] values = { + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + }; + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); } catch (Exception e) { e.printStackTrace(); @@ -127,7 +130,10 @@ public void sqlToArrowTest2() throws Exception { VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); - System.out.print(root.getRowCount()); + int[] values = { + 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118 + }; + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 18, values); } catch (Exception e) { e.printStackTrace(); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 181a05e626b..d1129391808 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -35,6 +35,8 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import static org.junit.Assert.*; + /** * This is a Helper class which has functionalities to read and print the values from FieldVector object @@ -42,16 +44,17 @@ */ public class JdbcToArrowTestHelper { - public void getIntVectorValues(FieldVector fx){ + public static boolean assertIntVectorValues(FieldVector fx, int rowCount, int[] values) { IntVector intVector = ((IntVector) fx); - for(int j = 0; j < intVector.getValueCount(); j++){ - if(!intVector.isNull(j)){ - int value = intVector.get(j); - System.out.println("Int Vector Value [" + j +"] : " + value); - } else { - System.out.println("Int Vector Value [" + j +"] : NULL "); + + assertEquals(rowCount, intVector.getValueCount()); + + for(int j = 0; j < intVector.getValueCount(); j++) { + if(!intVector.isNull(j)) { + assertEquals(values[j], intVector.get(j)); } } + return true; } public void getBigIntVectorValues(FieldVector fx){ From f5e95d26651ff1f6ade3d3f5aeb58c2da571d48b Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 16:41:46 -0700 Subject: [PATCH 033/129] Renamed a file. --- .../adapter/jdbc/src/test/resources/{test3_h2.yml => test_h2.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename java/adapter/jdbc/src/test/resources/{test3_h2.yml => test_h2.txt} (100%) diff --git a/java/adapter/jdbc/src/test/resources/test3_h2.yml b/java/adapter/jdbc/src/test/resources/test_h2.txt similarity index 100% rename from java/adapter/jdbc/src/test/resources/test3_h2.yml rename to java/adapter/jdbc/src/test/resources/test_h2.txt From 1dd9079190de2ca686a265516b2a309500b8966e Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 16:43:00 -0700 Subject: [PATCH 034/129] Renamed a file. --- .../jdbc/src/test/resources/{test1_h2.yml => test1_int_h2.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename java/adapter/jdbc/src/test/resources/{test1_h2.yml => test1_int_h2.yml} (100%) diff --git a/java/adapter/jdbc/src/test/resources/test1_h2.yml b/java/adapter/jdbc/src/test/resources/test1_int_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_h2.yml rename to java/adapter/jdbc/src/test/resources/test1_int_h2.yml From 5c56cf7a3ae3d86006d203d3f6d8aceff51bb6ed Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 18:18:58 -0700 Subject: [PATCH 035/129] Added assertion related test cases for various vector types. Added new test case files. --- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 73 ++++++- .../adapter/jdbc/JdbcToArrowTestHelper.java | 188 ++++++++++-------- .../test/resources/test1_all_datatypes_h2.yml | 58 ++++++ .../src/test/resources/test1_bigint_h2.yml | 57 ++++++ .../src/test/resources/test1_binary_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_bit_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_blob_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_bool_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_char_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_clob_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_date_h2.yml | 57 ++++++ .../src/test/resources/test1_decimal_h2.yml | 57 ++++++ .../src/test/resources/test1_double_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_real_h2.yml | 57 ++++++ .../src/test/resources/test1_smallint_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test1_time_h2.yml | 57 ++++++ .../src/test/resources/test1_timestamp_h2.yml | 57 ++++++ .../src/test/resources/test1_tinyint_h2.yml | 57 ++++++ .../src/test/resources/test1_varchar_h2.yml | 57 ++++++ .../jdbc/src/test/resources/test2_h2.yml | 28 --- 20 files changed, 1141 insertions(+), 118 deletions(-) create mode 100644 java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_binary_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_bit_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_blob_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_bool_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_char_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_clob_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_date_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_double_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_real_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_time_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml delete mode 100644 java/adapter/jdbc/src/test/resources/test2_h2.yml diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 2a5fa25dc9a..c615f0f0572 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.InputStream; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -118,11 +119,11 @@ public void sqlToArrowTest1() throws Exception { } @Test - public void sqlToArrowTest2() throws Exception { + public void sqlToArrowTestAllDataTypes() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test2_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("test1_all_datatypes_h2.yml"), Table.class); try { @@ -130,10 +131,72 @@ public void sqlToArrowTest2() throws Exception { VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); - int[] values = { - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118 + int[] ints = { + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 + }; + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); + + int[] bools = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + + int[] tinyints = { + 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 + }; + JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + + int[] smallints = { + 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 + }; + JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + + int[] bigints = { + 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, + 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 + }; + JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + + BigDecimal[] bigdecimals = { + new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), + new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), + new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), + new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), + new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) + }; + JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); + + double[] doubles = { + 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, + 56478356785.345, 56478356785.345, 56478356785.345, + 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 + }; + JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); + + float[] reals = { + 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, + 56478356785.345f, 56478356785.345f, 56478356785.345f, + 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f + }; + JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); + + int[] times = { + 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, + 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000 + }; + JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); + + long[] dates = { + 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, + 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l + }; + JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); + + long[] timestamps = { + 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, + 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 18, values); + JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); } catch (Exception e) { e.printStackTrace(); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index d1129391808..eeeb98323a4 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -57,139 +57,157 @@ public static boolean assertIntVectorValues(FieldVector fx, int rowCount, int[] return true; } - public void getBigIntVectorValues(FieldVector fx){ - BigIntVector bigIntVector = ((BigIntVector)fx); - for(int j = 0; j < bigIntVector.getValueCount(); j++){ - if(!bigIntVector.isNull(j)){ - long value = bigIntVector.get(j); - System.out.println("Big Int Vector Value [" + j +"] : " + value); - } else { - System.out.println("Big Int Vector Value [" + j +"] : NULL "); + public static boolean assertBitBooleanVectorValues(FieldVector fx, int rowCount, int[] values){ + BitVector bitVector = ((BitVector)fx); + assertEquals(rowCount, bitVector.getValueCount()); + for(int j = 0; j < bitVector.getValueCount(); j++){ + if(!bitVector.isNull(j)) { + assertEquals(values[j], bitVector.get(j)); } } + return true; } - public void getVarBinaryVectorValues(FieldVector fx){ - VarBinaryVector varBinaryVector =((VarBinaryVector) fx); - for(int j = 0; j < varBinaryVector.getValueCount(); j++){ - if(!varBinaryVector.isNull(j)){ - byte[] value = varBinaryVector.get(j); - long valHash = hashArray(value); - System.out.println("Var Binary Vector Value [" + j +"] : " + firstX(value, 5)); - } else { - System.out.println("Var Binary Vector Value [" + j +"] : NULL "); + public static boolean assertTinyIntVectorValues(FieldVector fx, int rowCount, int[] values){ + TinyIntVector tinyIntVector = ((TinyIntVector)fx); + + assertEquals(rowCount, tinyIntVector.getValueCount()); + + for(int j = 0; j < tinyIntVector.getValueCount(); j++){ + if(!tinyIntVector.isNull(j)) { + assertEquals(values[j], tinyIntVector.get(j)); } } + return true; } - public void getFloat4VectorValues(FieldVector fx){ - Float4Vector float4Vector = ((Float4Vector)fx); - for(int j = 0; j < float4Vector.getValueCount(); j++){ - if(!float4Vector.isNull(j)){ - float value = float4Vector.get(j); - System.out.println("Float Vector Value [" + j +"] : " + value); - } else { - System.out.println("Float Vector Value [" + j +"] : NULL "); + public static boolean assertSmallIntVectorValues(FieldVector fx, int rowCount, int[] values){ + SmallIntVector smallIntVector = ((SmallIntVector)fx); + + assertEquals(rowCount, smallIntVector.getValueCount()); + + for(int j = 0; j < smallIntVector.getValueCount(); j++){ + if(!smallIntVector.isNull(j)){ + assertEquals(values[j], smallIntVector.get(j)); } } + + return true; } - public void getFloat8VectorValues(FieldVector fx){ - Float8Vector float8Vector = ((Float8Vector)fx); - for(int j = 0; j < float8Vector.getValueCount(); j++){ - if(!float8Vector.isNull(j)){ - double value = float8Vector.get(j); - System.out.println("Double Vector Value [" + j +"] : " + value); - } else { - System.out.println("Double Vector Value [" + j +"] : NULL "); + public static boolean assertBigIntVectorValues(FieldVector fx, int rowCount, int[] values){ + BigIntVector bigIntVector = ((BigIntVector)fx); + + assertEquals(rowCount, bigIntVector.getValueCount()); + + for(int j = 0; j < bigIntVector.getValueCount(); j++){ + if(!bigIntVector.isNull(j)) { + assertEquals(values[j], bigIntVector.get(j)); } } + + return true; } - public void getBitBooleanVectorValues(FieldVector fx){ - BitVector bitVector = ((BitVector)fx); - for(int j = 0; j < bitVector.getValueCount(); j++){ - if(!bitVector.isNull(j)){ - int value = bitVector.get(j); - System.out.println("Bit Boolean Vector Value[" + j +"] : " + value); - } else { - System.out.println("Bit Boolean Vector Value[" + j +"] : NULL "); + public static boolean assertDecimalVectorValues(FieldVector fx, int rowCount, BigDecimal[] values){ + DecimalVector decimalVector = ((DecimalVector)fx); + + assertEquals(rowCount, decimalVector.getValueCount()); + + for(int j = 0; j < decimalVector.getValueCount(); j++){ + if(!decimalVector.isNull(j)){ + assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); } } + + return true; } - public void getTinyIntVectorValues(FieldVector fx){ - TinyIntVector tinyIntVector = ((TinyIntVector)fx); - for(int j = 0; j < tinyIntVector.getValueCount(); j++){ - if(!tinyIntVector.isNull(j)){ - byte value = tinyIntVector.get(j); - System.out.println("Tiny Int Vector Value[" + j +"] : " + value); - } else { - System.out.println("Tiny Int Vector Value[" + j +"] : NULL "); + public static boolean assertFloat8VectorValues(FieldVector fx, int rowCount, double[] values){ + Float8Vector float8Vector = ((Float8Vector)fx); + + assertEquals(rowCount, float8Vector.getValueCount()); + + for(int j = 0; j < float8Vector.getValueCount(); j++){ + if(!float8Vector.isNull(j)) { + assertEquals(values[j], float8Vector.get(j), 0.01); } } + + return true; } - public void getSmallIntVectorValues(FieldVector fx){ - SmallIntVector smallIntVector = ((SmallIntVector)fx); - for(int j = 0; j < smallIntVector.getValueCount(); j++){ - if(!smallIntVector.isNull(j)){ - short value = smallIntVector.get(j); - System.out.println("Small Int Vector Value[" + j +"] : " + value); - } else { - System.out.println("Small Int Vector Value[" + j +"] : NULL "); + public static boolean assertFloat4VectorValues(FieldVector fx, int rowCount, float[] values){ + Float4Vector float4Vector = ((Float4Vector)fx); + + assertEquals(rowCount, float4Vector.getValueCount()); + + for(int j = 0; j < float4Vector.getValueCount(); j++){ + if(!float4Vector.isNull(j)){ + assertEquals(values[j], float4Vector.get(j), 0.01); } } + + return true; } - public void getDecimalVectorValues(FieldVector fx){ - DecimalVector decimalVector = ((DecimalVector)fx); - for(int j = 0; j < decimalVector.getValueCount(); j++){ - if(!decimalVector.isNull(j)){ - BigDecimal value = decimalVector.getObject(j); - System.out.println("Decimal Vector Value[" + j +"] : " + value); - } else { - System.out.println("Decimal Vector Value[" + j +"] : NULL "); + public static boolean assertTimeVectorValues(FieldVector fx, int rowCount, int[] values){ + TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); + + assertEquals(rowCount, timeMilliVector.getValueCount()); + + for(int j = 0; j < timeMilliVector.getValueCount(); j++){ + if(!timeMilliVector.isNull(j)){ + assertEquals(values[j], timeMilliVector.get(j)); } } + + return true; } - public void getDateVectorValues(FieldVector fx){ + public static boolean assertDateVectorValues(FieldVector fx, int rowCount, long[] values){ DateMilliVector dateMilliVector = ((DateMilliVector)fx); + + assertEquals(rowCount, dateMilliVector.getValueCount()); + for(int j = 0; j < dateMilliVector.getValueCount(); j++){ if(!dateMilliVector.isNull(j)){ - long value = dateMilliVector.get(j); - System.out.println("Date Milli Vector Value[" + j +"] : " + value); - } else { - System.out.println("Date Milli Vector Value[" + j +"] : NULL "); + assertEquals(values[j], dateMilliVector.get(j)); } } - } - public void getTimeVectorValues(FieldVector fx){ - TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); - for(int j = 0; j < timeMilliVector.getValueCount(); j++){ - if(!timeMilliVector.isNull(j)){ - int value = timeMilliVector.get(j); - System.out.println("Time Milli Vector Value[" + j +"] : " + value); - } else { - System.out.println("Time Milli Vector Value[" + j +"] : NULL "); - } - } + return true; } - public void getTimeStampVectorValues(FieldVector fx){ + public static boolean assertTimeStampVectorValues(FieldVector fx, int rowCount, long[] values){ TimeStampVector timeStampVector = ((TimeStampVector)fx); + + assertEquals(rowCount, timeStampVector.getValueCount()); + for(int j = 0; j < timeStampVector.getValueCount(); j++){ if(!timeStampVector.isNull(j)){ - long value = timeStampVector.get(j); - System.out.println("Time Stamp Vector Value [" + j +"] : " + value); + assertEquals(values[j], timeStampVector.get(j)); + } + } + + return true; + } + + public void getVarBinaryVectorValues(FieldVector fx){ + VarBinaryVector varBinaryVector =((VarBinaryVector) fx); + for(int j = 0; j < varBinaryVector.getValueCount(); j++){ + if(!varBinaryVector.isNull(j)){ + byte[] value = varBinaryVector.get(j); + long valHash = hashArray(value); + System.out.println("Var Binary Vector Value [" + j +"] : " + firstX(value, 5)); } else { - System.out.println("Time Stamp Vector Value [" + j +"] : NULL "); + System.out.println("Var Binary Vector Value [" + j +"] : NULL "); } } } + + public void getVarcharVectorValues(FieldVector fx){ VarCharVector varCharVector = ((VarCharVector)fx); for(int j = 0; j < varCharVector.getValueCount(); j++){ diff --git a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml new file mode 100644 index 00000000000..fa712fba489 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml @@ -0,0 +1,58 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, + time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_binary_h2.yml b/java/adapter/jdbc/src/test/resources/test1_binary_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_binary_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_bit_h2.yml b/java/adapter/jdbc/src/test/resources/test1_bit_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_bit_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_blob_h2.yml b/java/adapter/jdbc/src/test/resources/test1_blob_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_blob_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml b/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_char_h2.yml b/java/adapter/jdbc/src/test/resources/test1_char_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_char_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_clob_h2.yml b/java/adapter/jdbc/src/test/resources/test1_clob_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_clob_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/test1_date_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_date_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml b/java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_double_h2.yml b/java/adapter/jdbc/src/test/resources/test1_double_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_double_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_real_h2.yml b/java/adapter/jdbc/src/test/resources/test1_real_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_real_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_time_h2.yml b/java/adapter/jdbc/src/test/resources/test1_time_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_time_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml b/java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml new file mode 100644 index 00000000000..338a554383e --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml @@ -0,0 +1,57 @@ + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +query: 'select int_field1 from table1;' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test2_h2.yml b/java/adapter/jdbc/src/test/resources/test2_h2.yml deleted file mode 100644 index 2f124f83d69..00000000000 --- a/java/adapter/jdbc/src/test/resources/test2_h2.yml +++ /dev/null @@ -1,28 +0,0 @@ - -name: 'table2' - -create: 'CREATE TABLE table2 (int_field1 INT);' - -data: - - 'INSERT INTO table2 VALUES (101);' - - 'INSERT INTO table2 VALUES (102);' - - 'INSERT INTO table2 VALUES (103);' - - 'INSERT INTO table2 VALUES (104);' - - 'INSERT INTO table2 VALUES (105);' - - 'INSERT INTO table2 VALUES (106);' - - 'INSERT INTO table2 VALUES (107);' - - 'INSERT INTO table2 VALUES (108);' - - 'INSERT INTO table2 VALUES (109);' - - 'INSERT INTO table2 VALUES (110);' - - 'INSERT INTO table2 VALUES (111);' - - 'INSERT INTO table2 VALUES (112);' - - 'INSERT INTO table2 VALUES (113);' - - 'INSERT INTO table2 VALUES (114);' - - 'INSERT INTO table2 VALUES (115);' - - 'INSERT INTO table2 VALUES (116);' - - 'INSERT INTO table2 VALUES (117);' - - 'INSERT INTO table2 VALUES (118);' - -query: 'select int_field1 from table2;' - -drop: 'DROP table table2;' \ No newline at end of file From eff855a1d162711d19d8df49c1d7f388248cf5b2 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 18:21:27 -0700 Subject: [PATCH 036/129] Fixed a typo in test file name. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index c615f0f0572..192f925ded5 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -97,7 +97,7 @@ public void sqlToArrowTest1() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), Table.class); try { From 230ba4aca47cb77970e0ff0dcd6c30bdfde5a3cc Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 20:20:50 -0700 Subject: [PATCH 037/129] Added test case for testing all the H2 data types in the same sql query. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 3 +- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 65 +++++++++++++++---- .../adapter/jdbc/JdbcToArrowTestHelper.java | 37 +++++++---- .../test/resources/test1_all_datatypes_h2.yml | 26 ++++++-- 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 38c3ec501f3..db904e6adb7 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -27,6 +27,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.nio.charset.Charset; import java.sql.*; import java.util.List; @@ -225,7 +226,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw String value = rs.getString(i) != null ? rs.getString(i) : ""; varcharVector.setIndexDefined(i); varcharVector.setValueLengthSafe(i, value.length()); - varcharVector.setSafe(rowCount, value.getBytes(), 0, value.length()); + varcharVector.setSafe(rowCount, value.getBytes(Charset.forName("UTF-8")), 0, value.length()); varcharVector.setValueCount(rowCount + 1); break; case Types.DATE: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 192f925ded5..bcebfcc1a97 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.InputStream; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -108,7 +109,7 @@ public void sqlToArrowTest1() throws Exception { int[] values = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 13, values); } catch (Exception e) { e.printStackTrace(); @@ -134,28 +135,28 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 13, ints); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 13, bools); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 13, tinyints); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 13, smallints); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 13, bigints); BigDecimal[] bigdecimals = { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), @@ -164,40 +165,80 @@ public void sqlToArrowTestAllDataTypes() throws Exception { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) }; - JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); + JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 13, bigdecimals); double[] doubles = { 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 }; - JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); + JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 13, doubles); float[] reals = { 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f }; - JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); + JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 13, reals); int[] times = { 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000 }; - JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); + JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 13, times); long[] dates = { 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l }; - JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); + JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 13, dates); long[] timestamps = { 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l }; - JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); + JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 13, timestamps); + + byte[][] bytes = { + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") + }; + JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 13, bytes); + + byte[] strb = "some text that needs to be converted to varchar".getBytes(); + byte[][] varchars = { + strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb + }; + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 13, varchars); + + JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 13, bytes); + + strb = "some text that needs to be converted to clob".getBytes(); + varchars = new byte[][] { + strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb + }; + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 13, varchars); + + strb = "some char text".getBytes(); + varchars = new byte[][] { + strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb + }; + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 13, varchars); + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 13, bools); } catch (Exception e) { e.printStackTrace(); } finally { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index eeeb98323a4..5dabef7df38 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -193,33 +193,32 @@ public static boolean assertTimeStampVectorValues(FieldVector fx, int rowCount, return true; } - public void getVarBinaryVectorValues(FieldVector fx){ + public static boolean assertVarBinaryVectorValues(FieldVector fx, int rowCount, byte[][] values){ VarBinaryVector varBinaryVector =((VarBinaryVector) fx); + + assertEquals(rowCount, varBinaryVector.getValueCount()); + for(int j = 0; j < varBinaryVector.getValueCount(); j++){ if(!varBinaryVector.isNull(j)){ - byte[] value = varBinaryVector.get(j); - long valHash = hashArray(value); - System.out.println("Var Binary Vector Value [" + j +"] : " + firstX(value, 5)); - } else { - System.out.println("Var Binary Vector Value [" + j +"] : NULL "); + assertEquals(hashArray(values[j]), hashArray(varBinaryVector.get(j))); } } + + return true; } + public static boolean assertVarcharVectorValues(FieldVector fx, int rowCount, byte[][] values) { + VarCharVector varCharVector = ((VarCharVector)fx); + assertEquals(rowCount, varCharVector.getValueCount()); - public void getVarcharVectorValues(FieldVector fx){ - VarCharVector varCharVector = ((VarCharVector)fx); for(int j = 0; j < varCharVector.getValueCount(); j++){ if(!varCharVector.isNull(j)){ - byte[] valArr = varCharVector.get(j); - //Text value = varCharVector.getObject(j); - //System.out.println("Var Char Vector Value [" + j +"] : " + value.toString()); - System.out.println("Var Char Accessor as byte[" + j +"] " + firstX(valArr, 5)); - } else { - System.out.println("\t\t varCharAccessor[" + j +"] : NULL "); + assertEquals(hashArray(values[j]), hashArray(varCharVector.get(j))); } } + + return true; } public static long hashArray(byte[] data){ @@ -238,4 +237,14 @@ public static String firstX(byte[] data, int items){ return sb.toString(); } + public static byte[] hexStringToByteArray(String s) { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + + Character.digit(s.charAt(i+1), 16)); + } + return data; + } + } diff --git a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml index fa712fba489..ffcac02ae28 100644 --- a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml @@ -9,48 +9,62 @@ data: - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +# +# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', +# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', +# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1;' From f95561414f9087a24b250b9c01ce660ea51d3679 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 12 Mar 2018 20:21:49 -0700 Subject: [PATCH 038/129] Fixed doc comment. --- .../org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 5dabef7df38..d85187eb1de 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -39,7 +39,7 @@ /** - * This is a Helper class which has functionalities to read and print the values from FieldVector object + * This is a Helper class which has functionalities to read and assert the values from teh given FieldVector object * */ public class JdbcToArrowTestHelper { From 6911c401d590e5f8227ebcddf324bddd1f230f99 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 10:45:22 -0700 Subject: [PATCH 039/129] Fixed a bug while creating VarBinary, VarChar vector types. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 20 +++++------ .../arrow/adapter/jdbc/JdbcToArrowTest.java | 34 +++++++++---------- .../test/resources/test1_all_datatypes_h2.yml | 14 ++++---- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index db904e6adb7..c29e8fc9100 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -224,8 +224,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.LONGVARCHAR: VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); String value = rs.getString(i) != null ? rs.getString(i) : ""; - varcharVector.setIndexDefined(i); - varcharVector.setValueLengthSafe(i, value.length()); + varcharVector.setIndexDefined(rowCount); + varcharVector.setValueLengthSafe(rowCount, value.length()); varcharVector.setSafe(rowCount, value.getBytes(Charset.forName("UTF-8")), 0, value.length()); varcharVector.setValueCount(rowCount + 1); break; @@ -268,8 +268,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw varBinaryVector.setValueCount(rowCount + 1); byte[] bytes = rs.getBytes(i); if (bytes != null) { - varBinaryVector.setIndexDefined(i); - varBinaryVector.setValueLengthSafe(i, bytes.length); + varBinaryVector.setIndexDefined(rowCount); + varBinaryVector.setValueLengthSafe(rowCount, bytes.length); varBinaryVector.setSafe(rowCount, bytes); } else { varBinaryVector.setNull(rowCount); @@ -285,9 +285,9 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw Clob clob = rs.getClob(i); if (clob != null) { int length = (int) clob.length(); - varcharVector1.setIndexDefined(i); - varcharVector1.setValueLengthSafe(i, length); - varcharVector1.setSafe(varcharVector1.getValueCapacity(), clob.getSubString(1, length).getBytes(), 0, length); + varcharVector1.setIndexDefined(rowCount); + varcharVector1.setValueLengthSafe(rowCount, length); + varcharVector1.setSafe(rowCount, clob.getSubString(1, length).getBytes(), 0, length); } else { varcharVector1.setNull(rowCount); } @@ -298,9 +298,9 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw Blob blob = rs.getBlob(i); if (blob != null) { byte[] data = blob.getBytes(0, (int) blob.length()); - varBinaryVector1.setIndexDefined(i); - varBinaryVector1.setValueLengthSafe(i, (int) blob.length()); - varBinaryVector1.setSafe(i, data); + varBinaryVector1.setIndexDefined(rowCount); + varBinaryVector1.setValueLengthSafe(rowCount, (int) blob.length()); + varBinaryVector1.setSafe(rowCount, data); } else { varBinaryVector1.setNull(rowCount);} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index bcebfcc1a97..4f8fb53d314 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -135,28 +135,28 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 13, ints); + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 13, bools); + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 13, tinyints); + JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 13, smallints); + JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 13, bigints); + JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); BigDecimal[] bigdecimals = { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), @@ -165,39 +165,39 @@ public void sqlToArrowTestAllDataTypes() throws Exception { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) }; - JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 13, bigdecimals); + JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); double[] doubles = { 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 }; - JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 13, doubles); + JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); float[] reals = { 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f }; - JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 13, reals); + JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); int[] times = { 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000 }; - JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 13, times); + JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); long[] dates = { 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l }; - JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 13, dates); + JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); long[] timestamps = { 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l }; - JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 13, timestamps); + JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); byte[][] bytes = { JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), @@ -216,29 +216,29 @@ public void sqlToArrowTestAllDataTypes() throws Exception { JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") }; - JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 13, bytes); + JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 15, bytes); byte[] strb = "some text that needs to be converted to varchar".getBytes(); byte[][] varchars = { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 13, varchars); + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 15, varchars); - JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 13, bytes); + JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 15, bytes); strb = "some text that needs to be converted to clob".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 13, varchars); + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 15, varchars); strb = "some char text".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 13, varchars); + JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 15, varchars); - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 13, bools); + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 15, bools); } catch (Exception e) { e.printStackTrace(); } finally { diff --git a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml index ffcac02ae28..513dae55293 100644 --- a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml @@ -58,13 +58,13 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -# -# - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', -# ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', -# ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', + ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1;' From f624bceeb196438670fbb532aa0d29775efe2686 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 10:53:46 -0700 Subject: [PATCH 040/129] Added test cases for bool, tinyint, smallint and bigint data types.. --- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 109 +++++++++++++++++- .../src/test/resources/test1_bigint_h2.yml | 2 +- .../jdbc/src/test/resources/test1_bool_h2.yml | 2 +- .../src/test/resources/test1_smallint_h2.yml | 2 +- .../src/test/resources/test1_tinyint_h2.yml | 2 +- 5 files changed, 111 insertions(+), 6 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 4f8fb53d314..79e2a9108ca 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -94,7 +94,7 @@ private void deleteTestData(Table table) throws Exception { } @Test - public void sqlToArrowTest1() throws Exception { + public void sqlToArrowTestInt() throws Exception { Table table = mapper.readValue( @@ -109,7 +109,112 @@ public void sqlToArrowTest1() throws Exception { int[] values = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 13, values); + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void sqlToArrowTestBool() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_bool_h2.yml"), + Table.class); + + try { + createTestData(table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + + int[] bools = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void sqlToArrowTestTinyInt() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_tinyint_h2.yml"), + Table.class); + + try { + createTestData(table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + + int[] tinyints = { + 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 + }; + JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void sqlToArrowTestSmallInt() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_smallint_h2.yml"), + Table.class); + + try { + createTestData(table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + + int[] smallints = { + 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 + }; + JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void sqlToArrowTestBigInt() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_bigint_h2.yml"), + Table.class); + + try { + createTestData(table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + + int[] bigints = { + 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, + 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 + }; + JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); } catch (Exception e) { e.printStackTrace(); diff --git a/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml index 338a554383e..c46de1f1591 100644 --- a/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml @@ -52,6 +52,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select bigint_field5 from table1;' drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml b/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml index 338a554383e..6396fe09887 100644 --- a/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml @@ -52,6 +52,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select bool_field2 from table1;' drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml index 338a554383e..910ae297b02 100644 --- a/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml @@ -52,6 +52,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select smallint_field4 from table1;' drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml b/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml index 338a554383e..864dcd6b221 100644 --- a/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml @@ -52,6 +52,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select tinyint_field3 from table1;' drop: 'DROP table table1;' \ No newline at end of file From 3bb513a68b73dd7fd1f2bf36340fa9d307d91e39 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 11:39:41 -0700 Subject: [PATCH 041/129] Added doc comment. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index c29e8fc9100..f6d48628d3e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -43,29 +43,34 @@ public class JdbcToArrowUtils { private static final int DEFAULT_BUFFER_SIZE = 256; /** - * JDBC type Java type - CHAR String - VARCHAR String - LONGVARCHAR String - NUMERIC java.math.BigDecimal - DECIMAL java.math.BigDecimal - BIT boolean - TINYINT byte - SMALLINT short - INTEGER int - BIGINT long - REAL float - FLOAT double - DOUBLE double - BINARY byte[] - VARBINARY byte[] - LONGVARBINARY byte[] - DATE java.sql.Date - TIME java.sql.Time - TIMESTAMP java.sql.Timestamp - + * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. + * + * This method current performs following mapping for JDBC SQL data types to corresponding Arrow data types. + * + * CHAR --> ArrowType.Utf8 + * VARCHAR --> ArrowType.Utf8 + * LONGVARCHAR --> ArrowType.Utf8 + * NUMERIC --> ArrowType.Decimal(precision, scale) + * DECIMAL --> ArrowType.Decimal(precision, scale) + * BIT --> ArrowType.Bool + * TINYINT --> ArrowType.Int(8, signed) + * SMALLINT --> ArrowType.Int(16, signed) + * INTEGER --> ArrowType.Int(32, signed) + * BIGINT --> ArrowType.Int(64, signed) + * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) + * BINARY --> ArrowType.Binary + * VARBINARY --> ArrowType.Binary + * LONGVARBINARY --> ArrowType.Binary + * DATE --> ArrowType.Date(DateUnit.MILLISECOND) + * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) + * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) + * CLOB --> ArrowType.Utf8 + * BLOB --> ArrowType.Binary + * * @param rsmd - * @return + * @return {@link Schema} * @throws SQLException */ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { @@ -158,6 +163,14 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { } } + /** + * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate + * the Arrow Vector obejcts. + * + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate + * @throws Exception + */ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { assert rs != null; @@ -315,8 +328,4 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw } root.setRowCount(rowCount); } - - - - } From 8e2576144a3ebf5a4faa2c382a892731afabc7ee Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 11:40:29 -0700 Subject: [PATCH 042/129] Added doc comment. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index f6d48628d3e..8ba56cd100e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -45,7 +45,7 @@ public class JdbcToArrowUtils { /** * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. * - * This method current performs following mapping for JDBC SQL data types to corresponding Arrow data types. + * This method currently performs following type mapping for JDBC SQL data types to corresponding Arrow data types. * * CHAR --> ArrowType.Utf8 * VARCHAR --> ArrowType.Utf8 From a007f3159c90adf2c6a9896631b76ffaf587b1c6 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 11:41:25 -0700 Subject: [PATCH 043/129] Fixed typo in doc comment. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 8ba56cd100e..01af12c95ce 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -165,7 +165,7 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { /** * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate - * the Arrow Vector obejcts. + * the given Arrow Vector objects. * * @param rs ResultSet to use to fetch the data from underlying database * @param root Arrow {@link VectorSchemaRoot} object to populate From 6641ad4995ab8e3590efe7b304f67775a0cb2ceb Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 12:33:14 -0700 Subject: [PATCH 044/129] Removed unused imports. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 79e2a9108ca..f94631fb282 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -25,10 +25,7 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.InputStream; import java.math.BigDecimal; -import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; From bffbe136d505632740ea9cefeedef3a2e80fe1a7 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 12:36:12 -0700 Subject: [PATCH 045/129] Added ArrowDataFetcher and Test classes. Added assertions for the API arguments. --- .../arrow/adapter/jdbc/ArrowDataFetcher.java | 57 +++++++++++++++++++ .../arrow/adapter/jdbc/JdbcToArrow.java | 17 ++++++ .../adapter/jdbc/ArrowDataFetcherTest.java | 23 ++++++++ 3 files changed, 97 insertions(+) create mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java new file mode 100644 index 00000000000..751dd72c0c9 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java @@ -0,0 +1,57 @@ +package org.apache.arrow.adapter.jdbc; + +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.sql.Connection; + +/** + * + */ +public class ArrowDataFetcher { + + private static final String all_columns_query = "select * from %s limit %d offset %d"; + private static final String custom_columns_query = "select %s from %s limit %d offset %d"; + private Connection connection; + private String tableName; + + public ArrowDataFetcher(Connection connection, String tableName) { + this.connection = connection; + this.tableName = tableName; + } + + public VectorSchemaRoot fetch(int limit, int offset, String... columns) throws Exception { + assert columns != null && columns.length > 0 : "columns can't be empty!"; + assert limit > 0 : "limit needs to be greater that 0"; + assert offset >= 0 : "offset needs to be greater than or equal to 0"; + + return JdbcToArrow.sqlToArrow(connection, + String.format(custom_columns_query, commaSeparatedQueryColumns(columns), + tableName, limit, offset)); + } + + public VectorSchemaRoot fetch(int limit, int offset) throws Exception { + assert limit > 0 : "limit needs to be greater that 0"; + assert offset >= 0 : "offset needs to be greater than or equal to 0"; + + return JdbcToArrow.sqlToArrow(connection, String.format(all_columns_query, tableName, limit, offset)); + } + + public static String commaSeparatedQueryColumns(String... columns) { + assert columns != null && columns.length > 0 : "columns can't be empty!"; + + StringBuilder columnBuilder = new StringBuilder(); + boolean insertComma = false; + for (String s: columns) { + if (insertComma) { + columnBuilder.append(','); + } + columnBuilder.append(' ').append(s); + insertComma = true; + } + columnBuilder.append(' '); + return columnBuilder.toString(); + } + +} + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 2799cf67678..b1f3eb775bd 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -39,6 +39,9 @@ public class JdbcToArrow { */ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { + assert connection != null: "JDBC conncetion object can not be null"; + assert query != null && query.length() > 0: "SQL query can not be null or empty"; + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); Statement stmt = null; @@ -64,6 +67,20 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query) t } } + /** + * This method returns ArrowDataFetcher Object that can be used to fetch and iterate on the data in the given + * database table. + * + * @param connection - Database connection Object + * @param tableName - Table name from which records will be fetched + * + * @return ArrowDataFetcher - Instance of ArrowDataFetcher which can be used to get Arrow Vector obejcts by calling its functionality + */ + public static ArrowDataFetcher jdbcArrowDataFetcher(Connection connection, String tableName) { + assert connection != null: "JDBC conncetion object can not be null"; + assert tableName != null && tableName.length() > 0: "Table name can not be null or empty"; + return new ArrowDataFetcher(connection, tableName); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java new file mode 100644 index 00000000000..47d63d9005e --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java @@ -0,0 +1,23 @@ +package org.apache.arrow.adapter.jdbc; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Test class for {@link ArrowDataFetcher}. + */ +public class ArrowDataFetcherTest { + + @Test + public void commaSeparatedQueryColumnsTest() { + try { + ArrowDataFetcher.commaSeparatedQueryColumns(null); + } catch (AssertionError error) { + assertTrue(true); + } + assertEquals(" one ", ArrowDataFetcher.commaSeparatedQueryColumns("one")); + assertEquals(" one, two ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two")); + assertEquals(" one, two, three ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two", "three")); + } +} From 29b40cdbf2c8d76987f0adff5f8b080ab756571d Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 12:39:26 -0700 Subject: [PATCH 046/129] Added ASF 2.0 license text. --- .../arrow/adapter/jdbc/ArrowDataFetcher.java | 18 ++++++++++++++++++ .../adapter/jdbc/ArrowDataFetcherTest.java | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java index 751dd72c0c9..2295411a16b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.arrow.adapter.jdbc; import org.apache.arrow.vector.VectorSchemaRoot; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java index 47d63d9005e..d2471b623b3 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.arrow.adapter.jdbc; import org.junit.Test; From f2c4ae003e9899d36d56474738bfaf1486c587f1 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 14:54:30 -0700 Subject: [PATCH 047/129] Added test cases for ArrowDataFetcher. Refactored test cases code. --- .../arrow/adapter/jdbc/ArrowDataFetcher.java | 4 +- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 71 +++++++++++++++++++ .../adapter/jdbc/ArrowDataFetcherTest.java | 69 +++++++++++++++++- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 61 +--------------- 4 files changed, 142 insertions(+), 63 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java index 2295411a16b..48a6e514dca 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java @@ -37,7 +37,7 @@ public ArrowDataFetcher(Connection connection, String tableName) { this.tableName = tableName; } - public VectorSchemaRoot fetch(int limit, int offset, String... columns) throws Exception { + public VectorSchemaRoot fetch(int offset, int limit, String... columns) throws Exception { assert columns != null && columns.length > 0 : "columns can't be empty!"; assert limit > 0 : "limit needs to be greater that 0"; assert offset >= 0 : "offset needs to be greater than or equal to 0"; @@ -47,7 +47,7 @@ public VectorSchemaRoot fetch(int limit, int offset, String... columns) throws E tableName, limit, offset)); } - public VectorSchemaRoot fetch(int limit, int offset) throws Exception { + public VectorSchemaRoot fetch(int offset, int limit) throws Exception { assert limit > 0 : "limit needs to be greater that 0"; assert offset >= 0 : "offset needs to be greater than or equal to 0"; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java new file mode 100644 index 00000000000..98c8d1ccd7e --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -0,0 +1,71 @@ +package org.apache.arrow.adapter.jdbc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.Before; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.Properties; + +/** + * Class to abstract out some common test functionality for testing JDBC to Arrow. + */ +public abstract class AbstractJdbcToArrowTest { + + protected Connection conn = null; + protected ObjectMapper mapper = null; + + @Before + public void setUp() throws Exception { + Properties properties = new Properties(); + properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + + mapper = new ObjectMapper(new YAMLFactory()); + + Class.forName(properties.getProperty("driver")); + + conn = DriverManager + .getConnection(properties.getProperty("url"), properties);; + } + + protected void createTestData(Table table) throws Exception { + + Statement stmt = null; + try { + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } + } + + } + + + protected void deleteTestData(Table table) throws Exception { + Statement stmt = null; + try { + stmt = conn.createStatement(); + stmt.executeUpdate(table.getDrop()); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java index d2471b623b3..320b2b8b5e2 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java @@ -18,6 +18,7 @@ package org.apache.arrow.adapter.jdbc; +import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.Test; import static org.junit.Assert.*; @@ -25,7 +26,7 @@ /** * Test class for {@link ArrowDataFetcher}. */ -public class ArrowDataFetcherTest { +public class ArrowDataFetcherTest extends AbstractJdbcToArrowTest { @Test public void commaSeparatedQueryColumnsTest() { @@ -38,4 +39,70 @@ public void commaSeparatedQueryColumnsTest() { assertEquals(" one, two ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two")); assertEquals(" one, two, three ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two", "three")); } + + @Test + public void arrowFetcherAllColumnsLimitOffsetTest() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), + Table.class); + + try { + createTestData(table); + + ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(super.conn, "table1"); + + VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10); + + int[] values = { + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 + }; + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 10, values); + + root = arrowDataFetcher.fetch(5, 5); + + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 5, values); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + @Test + public void arrowFetcherSingleColumnLimitOffsetTest() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), + Table.class); + + try { + createTestData(table); + + ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(super.conn, "table1"); + + VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10, "int_field1"); + + int[] values = { + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 + }; + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 10, values); + + root = arrowDataFetcher.fetch(5, 5); + + JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 5, values); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(table); + } + + } + + } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index f94631fb282..c18391ac216 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -18,77 +18,18 @@ package org.apache.arrow.adapter.jdbc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; -import java.util.Properties; /** * */ -public class JdbcToArrowTest { +public class JdbcToArrowTest extends AbstractJdbcToArrowTest { - private Connection conn = null; - private ObjectMapper mapper = null; - - @Before - public void setUp() throws Exception { - Properties properties = new Properties(); - properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); - - mapper = new ObjectMapper(new YAMLFactory()); - - Class.forName(properties.getProperty("driver")); - - conn = DriverManager - .getConnection(properties.getProperty("url"), properties);; - } - - private void createTestData(Table table) throws Exception { - - Statement stmt = null; - try { - //create the table and insert the data and once done drop the table - stmt = conn.createStatement(); - stmt.executeUpdate(table.getCreate()); - - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - stmt.close(); - } - } - - } - - - private void deleteTestData(Table table) throws Exception { - Statement stmt = null; - try { - stmt = conn.createStatement(); - stmt.executeUpdate(table.getDrop()); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - stmt.close(); - } - } - } @Test public void sqlToArrowTestInt() throws Exception { From d0e55f98a070d5447c434fdc066f60cac8fd4ed1 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 15:01:35 -0700 Subject: [PATCH 048/129] Added documentation comment. --- .../arrow/adapter/jdbc/ArrowDataFetcher.java | 32 +++++++++++++++++++ .../arrow/adapter/jdbc/JdbcToArrow.java | 3 ++ 2 files changed, 35 insertions(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java index 48a6e514dca..1670db9c550 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java @@ -23,7 +23,18 @@ import java.sql.Connection; /** + * Class to fetch data from a given database table where user can specify columns to fetch + * along with limit and offset parameters. * + * The object of this class is returned by invoking method jdbcArrowDataFetcher(Connection connection, String tableName) + * from {@link JdbcToArrow} class. Caller can use this object to fetch data repetitively based on the + * data fetch requirement and can implement pagination like functionality. + * + * This class doesn't hold any open connections to database but simply executes the "select" query everytime with + * the necessary limit and offset parameters. + * + * @since 0.10.0 + * @see JdbcToArrow */ public class ArrowDataFetcher { @@ -32,11 +43,25 @@ public class ArrowDataFetcher { private Connection connection; private String tableName; + /** + * Constructor + * @param connection + * @param tableName + */ public ArrowDataFetcher(Connection connection, String tableName) { this.connection = connection; this.tableName = tableName; } + /** + * Fetch the data from underlying table with the given limit and offset and for passed column names. + * + * @param offset + * @param limit + * @param columns + * @return + * @throws Exception + */ public VectorSchemaRoot fetch(int offset, int limit, String... columns) throws Exception { assert columns != null && columns.length > 0 : "columns can't be empty!"; assert limit > 0 : "limit needs to be greater that 0"; @@ -47,6 +72,13 @@ public VectorSchemaRoot fetch(int offset, int limit, String... columns) throws E tableName, limit, offset)); } + /** + * Fetch the data from underlying table with the given limit and offset and for all the columns. + * @param offset + * @param limit + * @return + * @throws Exception + */ public VectorSchemaRoot fetch(int offset, int limit) throws Exception { assert limit > 0 : "limit needs to be greater that 0"; assert offset >= 0 : "offset needs to be greater than or equal to 0"; diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index b1f3eb775bd..cc11d37e3fb 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -25,6 +25,9 @@ /** * Utility class to convert JDBC objects to columnar Arrow format objects. + * + * @since 0.10.0 + * @see ArrowDataFetcher */ public class JdbcToArrow { From f5f6efe56758ed33dbf03e0750f234f464b09466 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 15:03:51 -0700 Subject: [PATCH 049/129] Added documentation comment. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 24 +++++++++++++++++++ .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 ++ 2 files changed, 26 insertions(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index cc11d37e3fb..49a7b4f7601 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -26,6 +26,30 @@ /** * Utility class to convert JDBC objects to columnar Arrow format objects. * + * This utility uses following data mapping fto map JDBC/SQL datatype to Arrow data types. + * + * CHAR --> ArrowType.Utf8 + * VARCHAR --> ArrowType.Utf8 + * LONGVARCHAR --> ArrowType.Utf8 + * NUMERIC --> ArrowType.Decimal(precision, scale) + * DECIMAL --> ArrowType.Decimal(precision, scale) + * BIT --> ArrowType.Bool + * TINYINT --> ArrowType.Int(8, signed) + * SMALLINT --> ArrowType.Int(16, signed) + * INTEGER --> ArrowType.Int(32, signed) + * BIGINT --> ArrowType.Int(64, signed) + * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) + * BINARY --> ArrowType.Binary + * VARBINARY --> ArrowType.Binary + * LONGVARBINARY --> ArrowType.Binary + * DATE --> ArrowType.Date(DateUnit.MILLISECOND) + * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) + * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) + * CLOB --> ArrowType.Utf8 + * BLOB --> ArrowType.Binary + * * @since 0.10.0 * @see ArrowDataFetcher */ diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 01af12c95ce..a1c331508cc 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -36,7 +36,9 @@ /** + * Class that does most of the work to convert JDBC ResultSet data into Arrow columnar format Vector objects. * + * @since 0.10.0 */ public class JdbcToArrowUtils { From 8a218c47454fb04c5255b5fb1d6fa153fa84b4c6 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 17:57:47 -0700 Subject: [PATCH 050/129] Added ASF 2.0 license text. --- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 98c8d1ccd7e..279490ab6aa 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.arrow.adapter.jdbc; import com.fasterxml.jackson.databind.ObjectMapper; From 1441d3b50e48b361ab2a00a333a7779781bfec39 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 17:58:39 -0700 Subject: [PATCH 051/129] Added support for NCHAR, NVARCHAR and LONGNVARCHAR. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 3 +++ .../org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 49a7b4f7601..1a8d532758b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -29,8 +29,11 @@ * This utility uses following data mapping fto map JDBC/SQL datatype to Arrow data types. * * CHAR --> ArrowType.Utf8 + * NCHAR --> ArrowType.Utf8 * VARCHAR --> ArrowType.Utf8 + * NVARCHAR --> ArrowType.Utf8 * LONGVARCHAR --> ArrowType.Utf8 + * LONGNVARCHAR --> ArrowType.Utf8 * NUMERIC --> ArrowType.Decimal(precision, scale) * DECIMAL --> ArrowType.Decimal(precision, scale) * BIT --> ArrowType.Bool diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index a1c331508cc..8d0fd98339b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -50,8 +50,11 @@ public class JdbcToArrowUtils { * This method currently performs following type mapping for JDBC SQL data types to corresponding Arrow data types. * * CHAR --> ArrowType.Utf8 + * NCHAR --> ArrowType.Utf8 * VARCHAR --> ArrowType.Utf8 + * NVARCHAR --> ArrowType.Utf8 * LONGVARCHAR --> ArrowType.Utf8 + * LONGNVARCHAR --> ArrowType.Utf8 * NUMERIC --> ArrowType.Decimal(precision, scale) * DECIMAL --> ArrowType.Decimal(precision, scale) * BIT --> ArrowType.Bool @@ -114,8 +117,11 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(DOUBLE)), null)); break; case Types.CHAR: + case Types.NCHAR: case Types.VARCHAR: + case Types.NVARCHAR: case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); break; case Types.DATE: @@ -235,8 +241,11 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw float8Vector.setValueCount(rowCount + 1); break; case Types.CHAR: + case Types.NCHAR: case Types.VARCHAR: + case Types.NVARCHAR: case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); String value = rs.getString(i) != null ? rs.getString(i) : ""; varcharVector.setIndexDefined(rowCount); From 3b996218af7ca864e0fa3d58ff119bd826cf441e Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 17:59:42 -0700 Subject: [PATCH 052/129] Fixed a typo in doc comment. --- .../main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 1a8d532758b..a569bf1b78e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -26,7 +26,7 @@ /** * Utility class to convert JDBC objects to columnar Arrow format objects. * - * This utility uses following data mapping fto map JDBC/SQL datatype to Arrow data types. + * This utility uses following data mapping to map JDBC/SQL datatype to Arrow data types. * * CHAR --> ArrowType.Utf8 * NCHAR --> ArrowType.Utf8 From 5a5927867d935d1a2192ddc96d5eee9c48209cab Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 13 Mar 2018 18:50:31 -0700 Subject: [PATCH 053/129] Added JdbcToArrowTestHelper as static import. --- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index c18391ac216..875d367e425 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -25,6 +25,8 @@ import java.math.BigDecimal; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; + /** * */ @@ -47,7 +49,7 @@ public void sqlToArrowTestInt() throws Exception { int[] values = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); + assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); } catch (Exception e) { e.printStackTrace(); @@ -73,7 +75,7 @@ public void sqlToArrowTestBool() throws Exception { int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); } catch (Exception e) { e.printStackTrace(); @@ -99,7 +101,7 @@ public void sqlToArrowTestTinyInt() throws Exception { int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); } catch (Exception e) { e.printStackTrace(); @@ -125,7 +127,7 @@ public void sqlToArrowTestSmallInt() throws Exception { int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); } catch (Exception e) { e.printStackTrace(); @@ -152,7 +154,7 @@ public void sqlToArrowTestBigInt() throws Exception { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); } catch (Exception e) { e.printStackTrace(); @@ -178,28 +180,28 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); + assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - JdbcToArrowTestHelper.assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - JdbcToArrowTestHelper.assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - JdbcToArrowTestHelper.assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); BigDecimal[] bigdecimals = { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), @@ -208,80 +210,80 @@ public void sqlToArrowTestAllDataTypes() throws Exception { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) }; - JdbcToArrowTestHelper.assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); + assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); double[] doubles = { 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 }; - JdbcToArrowTestHelper.assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); + assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); float[] reals = { 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f }; - JdbcToArrowTestHelper.assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); + assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); int[] times = { 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000 }; - JdbcToArrowTestHelper.assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); + assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); long[] dates = { 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l }; - JdbcToArrowTestHelper.assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); + assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); long[] timestamps = { 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l }; - JdbcToArrowTestHelper.assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); + assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); byte[][] bytes = { - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - JdbcToArrowTestHelper.hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") }; - JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 15, bytes); + assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 15, bytes); byte[] strb = "some text that needs to be converted to varchar".getBytes(); byte[][] varchars = { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 15, varchars); + assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 15, varchars); - JdbcToArrowTestHelper.assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 15, bytes); + assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 15, bytes); strb = "some text that needs to be converted to clob".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 15, varchars); + assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 15, varchars); strb = "some char text".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - JdbcToArrowTestHelper.assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 15, varchars); + assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 15, varchars); - JdbcToArrowTestHelper.assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 15, bools); + assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 15, bools); } catch (Exception e) { e.printStackTrace(); } finally { From a6cc2fecaf15383083608346651d67f7361ff1f1 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 11:06:56 -0700 Subject: [PATCH 054/129] Fixed test cases related to time and timestamp to use the timezone. --- .../arrow/adapter/jdbc/JdbcToArrowTest.java | 10 ++-- .../adapter/jdbc/JdbcToArrowTestHelper.java | 2 +- .../test/resources/test1_all_datatypes_h2.yml | 60 +++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java index 875d367e425..1704fc07030 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java @@ -226,9 +226,9 @@ public void sqlToArrowTestAllDataTypes() throws Exception { }; assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); - int[] times = { - 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, - 74735000, 74735000, 74735000, 74735000, 74735000, 74735000, 74735000 + long[] times = { + 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, + 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000 }; assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); @@ -239,8 +239,8 @@ public void sqlToArrowTestAllDataTypes() throws Exception { assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); long[] timestamps = { - 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, - 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l, 1518468335000l + 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, + 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l }; assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index d85187eb1de..ef1fa39591e 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -151,7 +151,7 @@ public static boolean assertFloat4VectorValues(FieldVector fx, int rowCount, flo return true; } - public static boolean assertTimeVectorValues(FieldVector fx, int rowCount, int[] values){ + public static boolean assertTimeVectorValues(FieldVector fx, int rowCount, long[] values){ TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); assertEquals(rowCount, timeMilliVector.getValueCount()); diff --git a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml index 513dae55293..9d46aa5dc1c 100644 --- a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml @@ -6,64 +6,64 @@ create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', + PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, From 76c1efb2f6d4ff5533b0574b0538f4c8cf64dbbc Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 11:11:54 -0700 Subject: [PATCH 055/129] Moved H2 test cases under h2 directory. --- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 2 +- .../jdbc/{ => h2}/ArrowDataFetcherTest.java | 7 ++++--- .../jdbc/{ => h2}/JdbcToArrowTest.java | 20 ++++++++++--------- .../src/test/resources/{ => h2}/db.properties | 0 .../{ => h2}/test1_all_datatypes_h2.yml | 0 .../resources/{ => h2}/test1_bigint_h2.yml | 0 .../resources/{ => h2}/test1_binary_h2.yml | 0 .../test/resources/{ => h2}/test1_bit_h2.yml | 0 .../test/resources/{ => h2}/test1_blob_h2.yml | 0 .../test/resources/{ => h2}/test1_bool_h2.yml | 0 .../test/resources/{ => h2}/test1_char_h2.yml | 0 .../test/resources/{ => h2}/test1_clob_h2.yml | 0 .../test/resources/{ => h2}/test1_date_h2.yml | 0 .../resources/{ => h2}/test1_decimal_h2.yml | 0 .../resources/{ => h2}/test1_double_h2.yml | 0 .../test/resources/{ => h2}/test1_int_h2.yml | 0 .../test/resources/{ => h2}/test1_real_h2.yml | 0 .../resources/{ => h2}/test1_smallint_h2.yml | 0 .../test/resources/{ => h2}/test1_time_h2.yml | 0 .../resources/{ => h2}/test1_timestamp_h2.yml | 0 .../resources/{ => h2}/test1_tinyint_h2.yml | 0 .../resources/{ => h2}/test1_varchar_h2.yml | 0 .../src/test/resources/{ => h2}/test_h2.txt | 0 23 files changed, 16 insertions(+), 13 deletions(-) rename java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/{ => h2}/ArrowDataFetcherTest.java (95%) rename java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/{ => h2}/JdbcToArrowTest.java (96%) rename java/adapter/jdbc/src/test/resources/{ => h2}/db.properties (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_all_datatypes_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_bigint_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_binary_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_bit_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_blob_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_bool_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_char_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_clob_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_date_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_decimal_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_double_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_int_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_real_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_smallint_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_time_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_timestamp_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_tinyint_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test1_varchar_h2.yml (100%) rename java/adapter/jdbc/src/test/resources/{ => h2}/test_h2.txt (100%) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 279490ab6aa..47a3cbf869a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -38,7 +38,7 @@ public abstract class AbstractJdbcToArrowTest { @Before public void setUp() throws Exception { Properties properties = new Properties(); - properties.load(this.getClass().getClassLoader().getResourceAsStream("db.properties")); + properties.load(this.getClass().getClassLoader().getResourceAsStream("h2/db.properties")); mapper = new ObjectMapper(new YAMLFactory()); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java similarity index 95% rename from java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java rename to java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java index 320b2b8b5e2..b20caf9eb0e 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcherTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java @@ -16,8 +16,9 @@ * limitations under the License. */ -package org.apache.arrow.adapter.jdbc; +package org.apache.arrow.adapter.jdbc.h2; +import org.apache.arrow.adapter.jdbc.*; import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.Test; @@ -45,7 +46,7 @@ public void arrowFetcherAllColumnsLimitOffsetTest() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), Table.class); try { @@ -77,7 +78,7 @@ public void arrowFetcherSingleColumnLimitOffsetTest() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), Table.class); try { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java similarity index 96% rename from java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java rename to java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 1704fc07030..e5380260d13 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -16,11 +16,13 @@ * limitations under the License. */ -package org.apache.arrow.adapter.jdbc; +package org.apache.arrow.adapter.jdbc.h2; +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.math.BigDecimal; @@ -30,7 +32,7 @@ /** * */ -public class JdbcToArrowTest extends AbstractJdbcToArrowTest { +public class JdbcToArrowTest extends AbstractJdbcToArrowTest { @Test @@ -38,7 +40,7 @@ public void sqlToArrowTestInt() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_int_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), Table.class); try { @@ -64,7 +66,7 @@ public void sqlToArrowTestBool() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_bool_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_bool_h2.yml"), Table.class); try { @@ -90,7 +92,7 @@ public void sqlToArrowTestTinyInt() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_tinyint_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_tinyint_h2.yml"), Table.class); try { @@ -116,7 +118,7 @@ public void sqlToArrowTestSmallInt() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_smallint_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_smallint_h2.yml"), Table.class); try { @@ -142,7 +144,7 @@ public void sqlToArrowTestBigInt() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_bigint_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_bigint_h2.yml"), Table.class); try { @@ -169,7 +171,7 @@ public void sqlToArrowTestAllDataTypes() throws Exception { Table table = mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("test1_all_datatypes_h2.yml"), + this.getClass().getClassLoader().getResourceAsStream("h2/test1_all_datatypes_h2.yml"), Table.class); try { diff --git a/java/adapter/jdbc/src/test/resources/db.properties b/java/adapter/jdbc/src/test/resources/h2/db.properties similarity index 100% rename from java/adapter/jdbc/src/test/resources/db.properties rename to java/adapter/jdbc/src/test/resources/h2/db.properties diff --git a/java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_all_datatypes_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_bigint_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_binary_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_binary_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_bit_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_bit_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_blob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_blob_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_bool_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_bool_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_char_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_char_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_clob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_clob_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_date_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_decimal_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_double_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_double_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_int_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_int_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_real_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_real_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_smallint_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_time_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_timestamp_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_tinyint_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml similarity index 100% rename from java/adapter/jdbc/src/test/resources/test1_varchar_h2.yml rename to java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/test_h2.txt b/java/adapter/jdbc/src/test/resources/h2/test_h2.txt similarity index 100% rename from java/adapter/jdbc/src/test/resources/test_h2.txt rename to java/adapter/jdbc/src/test/resources/h2/test_h2.txt From 7f8f613fb60833ac09e336d0394e514428f0ec34 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 12:23:48 -0700 Subject: [PATCH 056/129] Fxied test case for date with timezone. --- .../adapter/jdbc/h2/JdbcToArrowTest.java | 4 +- .../resources/h2/test1_all_datatypes_h2.yml | 75 +++++++++++-------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index e5380260d13..c73659fe626 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -235,8 +235,8 @@ public void sqlToArrowTestAllDataTypes() throws Exception { assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); long[] dates = { - 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, - 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l, 1518422400000l + 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, + 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l }; assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 9d46aa5dc1c..0650e941bfd 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -6,64 +6,79 @@ create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), ''2018-02-12'', - PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, From 1d0f29b2ec6afd549b4438cd7d7739e91cdab3e3 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 12:50:51 -0700 Subject: [PATCH 057/129] Fxied test case for date with timezone. --- .../org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index c73659fe626..d6b41ab2507 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -235,8 +235,8 @@ public void sqlToArrowTestAllDataTypes() throws Exception { assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); long[] dates = { - 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, - 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l, 1518336000000l + 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, + 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l }; assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); From 8b52507b92396dd4e7ffb013fd1ae1628416969b Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 13:18:08 -0700 Subject: [PATCH 058/129] Added maven plugin to set timezone during Junit test execution. --- java/adapter/jdbc/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 8249a95f2ec..0b690d80b14 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -71,4 +71,19 @@ + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + UTC + + + + + + From 654325b3c1b400f9359f936aa5b5a5065107a237 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 14:52:33 -0700 Subject: [PATCH 059/129] Added *.yml extension. --- dev/release/rat_exclude_files.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index bf962bcd49e..6cc2d10799d 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -74,3 +74,5 @@ c_glib/doc/reference/gtk-doc.make *.svg *.devhelp2 *.scss +*.yml + From 63f773509f06c10b7a11b28846e11c6bd52e7489 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 16 Mar 2018 14:54:23 -0700 Subject: [PATCH 060/129] Added *.properties extension. --- dev/release/rat_exclude_files.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 6cc2d10799d..727033dde0e 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -75,4 +75,4 @@ c_glib/doc/reference/gtk-doc.make *.devhelp2 *.scss *.yml - +*.properties From 68c3a61d6f53f866a908e599b7983df83f81ef08 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Mon, 19 Mar 2018 14:04:13 -0700 Subject: [PATCH 061/129] Removed unused dependency. --- java/adapter/jdbc/pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 0b690d80b14..f615a62897b 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -55,19 +55,6 @@ 2.3.0 test - - - - - - - - - - - - - From 69d52028a58bb81db20f7240a8d9f10ff2fa4ca1 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Mar 2018 10:51:01 -0700 Subject: [PATCH 062/129] Added license notice header to YAML and PROPERTIES files instead of excluding these files from the rat check.. --- java/adapter/jdbc/pom.xml | 6 +++--- java/adapter/jdbc/src/test/resources/h2/db.properties | 11 +++++++++++ .../src/test/resources/h2/test1_all_datatypes_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_bigint_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_binary_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_bit_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_blob_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_bool_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_char_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_clob_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_date_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_decimal_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_double_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_int_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_real_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_smallint_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_time_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_timestamp_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_tinyint_h2.yml | 10 ++++++++++ .../jdbc/src/test/resources/h2/test1_varchar_h2.yml | 10 ++++++++++ java/pom.xml | 4 +--- 21 files changed, 195 insertions(+), 6 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index f615a62897b..35892483c28 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -17,7 +17,7 @@ org.apache.arrow.adapter.jdbc arrow-jdbc jar - 0.10.0-SNAPSHOT + 0.9.0-SNAPSHOT Arrow JDBC Adapter http://maven.apache.org @@ -26,13 +26,13 @@ org.apache.arrow arrow-memory - 0.9.0-SNAPSHOT + ${project.version} org.apache.arrow arrow-vector - 0.9.0-SNAPSHOT + ${project.version} diff --git a/java/adapter/jdbc/src/test/resources/h2/db.properties b/java/adapter/jdbc/src/test/resources/h2/db.properties index 52c67cd72c7..b10a78d74b2 100644 --- a/java/adapter/jdbc/src/test/resources/h2/db.properties +++ b/java/adapter/jdbc/src/test/resources/h2/db.properties @@ -1,3 +1,14 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + driver=org.h2.Driver url=jdbc:h2:~/test user=sa diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 0650e941bfd..f610a80d376 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml index c46de1f1591..88e3053a7d2 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml index 6396fe09887..ea0956996b4 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml index 910ae297b02..5fde24c5761 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml index 864dcd6b221..92c89967348 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml index 338a554383e..d360773c55f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml @@ -1,3 +1,13 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. name: 'table1' diff --git a/java/pom.xml b/java/pom.xml index 363bea54936..935df6a08e7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -118,7 +118,6 @@ **/*.proto **/*.fmpp **/target/** - **/*.iml **/*.tdd **/*.project **/TAGS @@ -132,8 +131,7 @@ **/*.linux **/client/build/** **/*.tbl - **/*.yml - **/*.properties + **/*.iml From 1400fc870593bb3d0b6a1f30530c8c18bd0f3c47 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Mar 2018 10:55:33 -0700 Subject: [PATCH 063/129] Removed earlier added YAML and PROPERTIES extensions from exclusions. --- dev/release/rat_exclude_files.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 727033dde0e..bf962bcd49e 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -74,5 +74,3 @@ c_glib/doc/reference/gtk-doc.make *.svg *.devhelp2 *.scss -*.yml -*.properties From fe1f27d5616b4ab2eeb15c967d5ba0f089c9a3f8 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Mar 2018 14:43:37 -0700 Subject: [PATCH 064/129] Fixed compilation errors in PR build due to project version. Fixed H2 DB creation for each Unit test case class. Removed Google util class usage and used JDK List. --- java/adapter/jdbc/pom.xml | 25 ++++++-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 7 ++- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 27 +-------- .../adapter/jdbc/h2/ArrowDataFetcherTest.java | 42 +++++++++++-- .../adapter/jdbc/h2/JdbcToArrowTest.java | 59 +++++++++++++------ .../jdbc/src/test/resources/h2/db.properties | 18 ------ 6 files changed, 102 insertions(+), 76 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/resources/h2/db.properties diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 35892483c28..b55e77243cb 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -14,10 +14,14 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - org.apache.arrow.adapter.jdbc + + + org.apache.arrow + arrow-java-root + 0.9.0-SNAPSHOT + + arrow-jdbc - jar - 0.9.0-SNAPSHOT Arrow JDBC Adapter http://maven.apache.org @@ -52,10 +56,21 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.3.0 + 2.7.9 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.7.9 + test + + + com.google.guava + guava + 18.0 test - diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 8d0fd98339b..9b201a872bf 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -18,7 +18,6 @@ package org.apache.arrow.adapter.jdbc; -import com.google.common.collect.ImmutableList; import org.apache.arrow.vector.*; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.TimeUnit; @@ -29,6 +28,7 @@ import java.nio.charset.Charset; import java.sql.*; +import java.util.ArrayList; import java.util.List; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; @@ -82,7 +82,8 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti assert rsmd != null; - ImmutableList.Builder fields = ImmutableList.builder(); +// ImmutableList.Builder fields = ImmutableList.builder(); + List fields = new ArrayList<>(); int columnCount = rsmd.getColumnCount(); for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); @@ -156,7 +157,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti } } - return new Schema(fields.build(), null); + return new Schema(fields, null); } private static void allocateVectors(VectorSchemaRoot root, int size) { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 47a3cbf869a..099f304abb3 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -18,37 +18,15 @@ package org.apache.arrow.adapter.jdbc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.junit.Before; - import java.sql.Connection; -import java.sql.DriverManager; import java.sql.Statement; -import java.util.Properties; /** * Class to abstract out some common test functionality for testing JDBC to Arrow. */ public abstract class AbstractJdbcToArrowTest { - protected Connection conn = null; - protected ObjectMapper mapper = null; - - @Before - public void setUp() throws Exception { - Properties properties = new Properties(); - properties.load(this.getClass().getClassLoader().getResourceAsStream("h2/db.properties")); - - mapper = new ObjectMapper(new YAMLFactory()); - - Class.forName(properties.getProperty("driver")); - - conn = DriverManager - .getConnection(properties.getProperty("url"), properties);; - } - - protected void createTestData(Table table) throws Exception { + protected void createTestData(Connection conn, Table table) throws Exception { Statement stmt = null; try { @@ -71,7 +49,7 @@ protected void createTestData(Table table) throws Exception { } - protected void deleteTestData(Table table) throws Exception { + protected void deleteTestData(Connection conn, Table table) throws Exception { Statement stmt = null; try { stmt = conn.createStatement(); @@ -85,5 +63,4 @@ protected void deleteTestData(Table table) throws Exception { } } } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java index b20caf9eb0e..b7fca36818f 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java @@ -18,10 +18,17 @@ package org.apache.arrow.adapter.jdbc.h2; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.arrow.adapter.jdbc.*; import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import java.sql.Connection; +import java.sql.DriverManager; + import static org.junit.Assert.*; /** @@ -29,6 +36,29 @@ */ public class ArrowDataFetcherTest extends AbstractJdbcToArrowTest { + private Connection conn = null; + private ObjectMapper mapper = null; + + @Before + public void setUp() throws Exception { + String url = "jdbc:h2:mem:ArrowDataFetcherTest"; + String driver = "org.h2.Driver"; + + mapper = new ObjectMapper(new YAMLFactory()); + + Class.forName(driver); + + conn = DriverManager.getConnection(url); + } + + @After + public void destroy() throws Exception { + if (conn != null) { + conn.close(); + conn = null; + } + } + @Test public void commaSeparatedQueryColumnsTest() { try { @@ -50,9 +80,9 @@ public void arrowFetcherAllColumnsLimitOffsetTest() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); - ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(super.conn, "table1"); + ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(conn, "table1"); VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10); @@ -68,7 +98,7 @@ public void arrowFetcherAllColumnsLimitOffsetTest() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -82,9 +112,9 @@ public void arrowFetcherSingleColumnLimitOffsetTest() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); - ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(super.conn, "table1"); + ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(conn, "table1"); VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10, "int_field1"); @@ -100,7 +130,7 @@ public void arrowFetcherSingleColumnLimitOffsetTest() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index d6b41ab2507..89558d961f0 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -18,14 +18,20 @@ package org.apache.arrow.adapter.jdbc.h2; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; +import org.junit.Before; import org.junit.Test; import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; @@ -34,6 +40,28 @@ */ public class JdbcToArrowTest extends AbstractJdbcToArrowTest { + private Connection conn = null; + private ObjectMapper mapper = null; + + @Before + public void setUp() throws Exception { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + + mapper = new ObjectMapper(new YAMLFactory()); + + Class.forName(driver); + + conn = DriverManager.getConnection(url); + } + + @After + public void destroy() throws Exception { + if (conn != null) { + conn.close(); + conn = null; + } + } @Test public void sqlToArrowTestInt() throws Exception { @@ -44,7 +72,7 @@ public void sqlToArrowTestInt() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -56,7 +84,7 @@ public void sqlToArrowTestInt() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -70,7 +98,7 @@ public void sqlToArrowTestBool() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -82,7 +110,7 @@ public void sqlToArrowTestBool() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -96,7 +124,7 @@ public void sqlToArrowTestTinyInt() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -108,7 +136,7 @@ public void sqlToArrowTestTinyInt() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -122,7 +150,7 @@ public void sqlToArrowTestSmallInt() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -134,7 +162,7 @@ public void sqlToArrowTestSmallInt() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -148,7 +176,7 @@ public void sqlToArrowTestBigInt() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -161,7 +189,7 @@ public void sqlToArrowTestBigInt() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } @@ -175,7 +203,7 @@ public void sqlToArrowTestAllDataTypes() throws Exception { Table.class); try { - createTestData(table); + createTestData(conn, table); VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); @@ -289,16 +317,9 @@ public void sqlToArrowTestAllDataTypes() throws Exception { } catch (Exception e) { e.printStackTrace(); } finally { - deleteTestData(table); + deleteTestData(conn, table); } } - @After - public void destroy() throws Exception { - if (conn != null) { - conn.close(); - } - } - } diff --git a/java/adapter/jdbc/src/test/resources/h2/db.properties b/java/adapter/jdbc/src/test/resources/h2/db.properties deleted file mode 100644 index b10a78d74b2..00000000000 --- a/java/adapter/jdbc/src/test/resources/h2/db.properties +++ /dev/null @@ -1,18 +0,0 @@ -#Licensed to the Apache Software Foundation (ASF) under one or more contributor -#license agreements. See the NOTICE file distributed with this work for additional -#information regarding copyright ownership. The ASF licenses this file to -#You under the Apache License, Version 2.0 (the "License"); you may not use -#this file except in compliance with the License. You may obtain a copy of -#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required -#by applicable law or agreed to in writing, software distributed under the -#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -#OF ANY KIND, either express or implied. See the License for the specific -#language governing permissions and limitations under the License. - -driver=org.h2.Driver -url=jdbc:h2:~/test -user=sa -password= - - - From c5b3f560d5afa630bd1453d915660046eaa5e5ac Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Mar 2018 15:53:54 -0700 Subject: [PATCH 065/129] Updated source from remote upstream for version 0.10.0-SNAPSHOT --- java/adapter/jdbc/pom.xml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index b55e77243cb..e132a628920 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -10,15 +10,12 @@ OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - - + 4.0.0 - org.apache.arrow arrow-java-root - 0.9.0-SNAPSHOT + 0.10.0-SNAPSHOT arrow-jdbc From 00d2f117b5c571340b0e4fa7dbae448a327a5123 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Fri, 23 Mar 2018 16:02:05 -0700 Subject: [PATCH 066/129] Removed a .txt file which was causing rat failure. --- .../jdbc/src/test/resources/h2/test_h2.txt | 96 ------------------- 1 file changed, 96 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/resources/h2/test_h2.txt diff --git a/java/adapter/jdbc/src/test/resources/h2/test_h2.txt b/java/adapter/jdbc/src/test/resources/h2/test_h2.txt deleted file mode 100644 index 845b5ba717f..00000000000 --- a/java/adapter/jdbc/src/test/resources/h2/test_h2.txt +++ /dev/null @@ -1,96 +0,0 @@ -#--- Created TABLE -1 : -CREATE TABLE H2Table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT, identity_field18 IDENTITY, varchar_ignorecase_field19 VARCHAR_IGNORECASE(30), UUID_field20 UUID, float_field21 FLOAT); - -#--- Inserted values in TABLE -1 : -INSERT INTO H2Table1 VALUES (101, 1, 41, 12001, 92233721, 17345667781.23, 56478356781.345, 56478356781.345, '1:45:35', '2018-02-1', '2018-02-1 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as varchar1', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as clob1', 'some char text1', 1, 10001, 'varchar_ignorecase_1', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); -INSERT INTO H2Table1 VALUES (102, 0, 42, 12002, 92233722, 17345667782.23, 56478356782.345, 56478356782.345, '2:45:35', '2018-02-2', '2018-02-2 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as varchar2', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as clob2', 'some char text2', 0, 10002, 'varchar_ignorecase_2', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); -INSERT INTO H2Table1 VALUES (103, 1, 43, 12003, 92233723, 17345667783.23, 56478356783.345, 56478356783.345, '3:45:35', '2018-02-3', '2018-02-3 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as varchar3', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as clob3', 'some char text3', 1, 10003, 'varchar_ignorecase_3', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13'); -INSERT INTO H2Table1 VALUES (104, 0, 44, 12004, 92233724, 17345667784.23, 56478356784.345, 56478356784.345, '4:45:35', '2018-02-4', '2018-02-4 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some as varchar4', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some text as clob4', 'some char text4', 0, 10004, 'varchar_ignorecase_4', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14'); -INSERT INTO H2Table1 VALUES (105, 1, 45, 12005, 92233725, 17345667785.23, 56478356785.345, 56478356785.345, '5:45:35', '2018-02-5', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as varchar5', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob5', 'some char text5', 1, 10005, 'varchar_ignorecase_5', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15'); - -#--- Created TABLE -2 : -CREATE TABLE H2Table2 (int_T2field1 INT, bool_T2field2 BOOLEAN, tinyint_T2field3 TINYINT, smallint_T2field4 SMALLINT, bigint_T2field5 BIGINT, decimal_T2field6 DECIMAL(20,2), double_T2field7 DOUBLE, real_T2field8 REAL, time_T2field9 TIME, date_T2field10 DATE, timestamp_T2field11 TIMESTAMP, binary_T2field12 BINARY(100), varchar_T2field13 VARCHAR(256), blob_T2field14 BLOB, clob_T2field15 CLOB, char_T2field16 CHAR(16), bit_T2field17 BIT, identity_T2field18 IDENTITY, varchar_ignorecase_T2field19 VARCHAR_IGNORECASE(30), UUID_T2field20 UUID, float_T2field21 FLOAT); - -#--- Inserted values in TABLE -2 : -INSERT INTO H2Table2 VALUES (106, 1, 46, 12006, 92233726, 17345667786.23, 56478356786.345, 56478356786.345, '6:45:35', '2018-02-6', '2018-02-6 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as varchar6', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as clob6', 'some char text6', 0, 10006, 'varchar_ignorecase_6', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16'); -INSERT INTO H2Table2 VALUES (107, 0, 47, 12007, 92233727, 17345667787.23, 56478356787.345, 56478356787.345, '7:45:35', '2018-02-7','2018-02-7 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as varchar7', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as clob7', 'some char text7', 1, 10007, 'varchar_ignorecase_7', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17'); -INSERT INTO H2Table2 VALUES (108, 1, 48, 02008, 92233728, 17345667788.23, 56478356788.345, 56478356788.345, '8:45:35', '2018-02-8', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as varchar8', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as clob8', 'some char text8', 1, 10008, 'varchar_ignorecase_8', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18'); -INSERT INTO H2Table2 VALUES (109, 0, 49, 12009, 92233729, 17345667789.23, 56478356789.345, 56478356789.345, '9:45:35', '2018-02-9', '2018-02-9 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as varchar9', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as clob9', 'some char text9', 0, 10009, 'varchar_ignorecase_9', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a19'); -INSERT INTO H2Table2 VALUES (110, 1, 50, 12010, 92233710, 17345667710.23, 56478356710.345, 56478356710.345, '10:45:35', '2018-02-10', '2018-02-10 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617210', 'some text as varchar10', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob10', 'some char text10', 1, 10010, 'varchar_ignorecase_10', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a10'); - -#--- Created TABLE -3 : -CREATE TABLE H2Table4 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); - ---- Inserted values in TABLE -3 : -INSERT INTO H2Table4 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); -INSERT INTO H2Table4 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); -INSERT INTO H2Table4 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); -INSERT INTO H2Table4 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); -INSERT INTO H2Table4 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); -INSERT INTO H2Table4 VALUES (214, 92233214, 17345667214.23, 56478356214.345, 'some text as varchar214'); - -#--- Created TABLE -4 : -CREATE TABLE H2Table5 (int_T5field1 INT, bigint_T5field2 BIGINT, decimal_T5field3 DECIMAL(20,2), float_T5field4 FLOAT, varchar_T5field5 VARCHAR(256)); - -#--- Inserted values in TABLE -4 : -INSERT INTO H2Table5 VALUES (206, 92233206, 17345667206.23, 56478356206.345, 'some text as varchar206'); -INSERT INTO H2Table5 VALUES (207, 92233207, 17345667207.23, 56478356207.345, 'some text as varchar207'); -INSERT INTO H2Table5 VALUES (208, 92233208, 17345667208.23, 56478356208.345, 'some text as varchar208'); -INSERT INTO H2Table5 VALUES (209, 92233209, 17345667209.23, 56478356209.345, 'some text as varchar209'); -INSERT INTO H2Table5 VALUES (210, 92233210, 17345667210.23, 56478356210.345, 'some text as varchar210'); - -#--- Created TABLE -5 : -CREATE TABLE H2Table6 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); - -#--- Inserted values in TABLE -5 : -INSERT INTO H2Table6 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); -INSERT INTO H2Table6 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); -INSERT INTO H2Table6 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); -INSERT INTO H2Table6 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); -INSERT INTO H2Table6 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); -INSERT INTO H2Table6 VALUES (212, 92233212, 17345667212.23, 56478356212.345, 'some text as varchar212'); - -#----------------------------------------------------------------------------------------------------------------------------------- - - -#---- JOINS QUERIES -SELECT * from H2Table4 AS T4 INNER JOIN H2Table6 AS T6 ON T4.int_T4field1 = T6.int_T4field1; -SELECT * from H2Table4 AS T4 RIGHT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; -SELECT * from H2Table4 AS T4 LEFT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; -SELECT * from H2Table4 CROSS JOIN H2Table6; -SELECT * from H2Table6 CROSS JOIN H2Table5; - -#---- SET/UNION OPERATIONS QUERIES -SELECT int_T4field1 FROM H2Table4 UNION SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 UNION ALL SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 INTERSECT SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 MINUS SELECT int_T4field1 FROM H2Table6; - -#---- ORDER BY CLAUSE QUERIES -SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 ASC; -SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 DESC; - -#---- GROUP BY CLAUSE QUERIES -SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2; - -#---- HAVING CLAUSE QUERIES -SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2 -HAVING h2t1.bool_field2 IS TRUE AND h2t1.tinyint_field3 > 41; - -#---- CASE STATEMENT QUERY -SELECT int_field1, BOOL_FIELD2, tinyint_field3, CASE WHEN tinyint_field3 > 44 THEN TRUE ELSE FALSE END FROM H2Table1; - -#---- AGGREGATE FUNCTIONS QUERIES -SELECT MAX(int_field1) AS Int_Val, MAX(tinyint_field3) AS TinyInt_Val, MAX(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT MIN(int_field1) AS Int_Val, MIN(tinyint_field3) AS TinyInt_Val, MIN(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT AVG(int_field1) AS Int_Val, AVG(tinyint_field3) AS TinyInt_Val, AVG(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT SUM(int_field1) AS Int_Val, SUM(tinyint_field3) AS TinyInt_Val, SUM(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT COUNT(int_field1) AS Int_Val, COUNT(tinyint_field3) AS TinyInt_Val, COUNT(smallint_field4) AS SmallInt_Val FROM H2Table1; - -SELECT char_field16 FROM H2Table1; -SELECT UCASE (char_field16) AS Char_Val, UCASE (varchar_field13) AS VarChar_Val FROM H2Table1; -SELECT LCASE (char_field16) AS Char_Val, LCASE (varchar_field13) AS VarChar_Val FROM H2Table1; - -SELECT ROUND (decimal_field6, 2) AS Decimal_Val, ROUND (float_field21, 1) Float_Val, -ROUND (double_field7, 3) AS Double_Val, ROUND (real_field8, 4) AS Real_Val FROM H2Table1; - From 6556a4433ef26fc1bd2e766b29e756456545d87b Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 3 Apr 2018 19:14:35 -0700 Subject: [PATCH 067/129] Code changes based on PR review comments. --- java/adapter/jdbc/pom.xml | 13 +- .../arrow/adapter/jdbc/ArrowDataFetcher.java | 107 ------- .../arrow/adapter/jdbc/JdbcToArrow.java | 79 +++-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 290 ++++++++++++------ .../org/apache/arrow/adapter/jdbc/Table.java | 2 +- .../adapter/jdbc/h2/ArrowDataFetcherTest.java | 139 --------- .../adapter/jdbc/h2/JdbcToArrowTest.java | 25 +- 7 files changed, 254 insertions(+), 401 deletions(-) delete mode 100644 java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index e132a628920..7913220bf11 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -35,6 +35,12 @@ arrow-vector ${project.version} + + com.google.guava + guava + 18.0 + + @@ -62,10 +68,11 @@ 2.7.9 test + - com.google.guava - guava - 18.0 + com.google.collections + google-collections + 1.0 test diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java deleted file mode 100644 index 1670db9c550..00000000000 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.adapter.jdbc; - -import org.apache.arrow.vector.VectorSchemaRoot; - -import java.sql.Connection; - -/** - * Class to fetch data from a given database table where user can specify columns to fetch - * along with limit and offset parameters. - * - * The object of this class is returned by invoking method jdbcArrowDataFetcher(Connection connection, String tableName) - * from {@link JdbcToArrow} class. Caller can use this object to fetch data repetitively based on the - * data fetch requirement and can implement pagination like functionality. - * - * This class doesn't hold any open connections to database but simply executes the "select" query everytime with - * the necessary limit and offset parameters. - * - * @since 0.10.0 - * @see JdbcToArrow - */ -public class ArrowDataFetcher { - - private static final String all_columns_query = "select * from %s limit %d offset %d"; - private static final String custom_columns_query = "select %s from %s limit %d offset %d"; - private Connection connection; - private String tableName; - - /** - * Constructor - * @param connection - * @param tableName - */ - public ArrowDataFetcher(Connection connection, String tableName) { - this.connection = connection; - this.tableName = tableName; - } - - /** - * Fetch the data from underlying table with the given limit and offset and for passed column names. - * - * @param offset - * @param limit - * @param columns - * @return - * @throws Exception - */ - public VectorSchemaRoot fetch(int offset, int limit, String... columns) throws Exception { - assert columns != null && columns.length > 0 : "columns can't be empty!"; - assert limit > 0 : "limit needs to be greater that 0"; - assert offset >= 0 : "offset needs to be greater than or equal to 0"; - - return JdbcToArrow.sqlToArrow(connection, - String.format(custom_columns_query, commaSeparatedQueryColumns(columns), - tableName, limit, offset)); - } - - /** - * Fetch the data from underlying table with the given limit and offset and for all the columns. - * @param offset - * @param limit - * @return - * @throws Exception - */ - public VectorSchemaRoot fetch(int offset, int limit) throws Exception { - assert limit > 0 : "limit needs to be greater that 0"; - assert offset >= 0 : "offset needs to be greater than or equal to 0"; - - return JdbcToArrow.sqlToArrow(connection, String.format(all_columns_query, tableName, limit, offset)); - } - - public static String commaSeparatedQueryColumns(String... columns) { - assert columns != null && columns.length > 0 : "columns can't be empty!"; - - StringBuilder columnBuilder = new StringBuilder(); - boolean insertComma = false; - for (String s: columns) { - if (insertComma) { - columnBuilder.append(','); - } - columnBuilder.append(' ').append(s); - insertComma = true; - } - columnBuilder.append(' '); - return columnBuilder.toString(); - } - -} - - diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index a569bf1b78e..e375a0264a8 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -21,7 +21,12 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import java.sql.*; +import com.google.common.base.Preconditions; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.ResultSet; /** * Utility class to convert JDBC objects to columnar Arrow format objects. @@ -54,7 +59,6 @@ * BLOB --> ArrowType.Binary * * @since 0.10.0 - * @see ArrowDataFetcher */ public class JdbcToArrow { @@ -64,53 +68,48 @@ public class JdbcToArrow { * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. - * @return - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception { - - assert connection != null: "JDBC conncetion object can not be null"; - assert query != null && query.length() > 0: "SQL query can not be null or empty"; - - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); - Statement stmt = null; - ResultSet rs = null; - try { - stmt = connection.createStatement(); - rs = stmt.executeQuery(query); - ResultSetMetaData rsmd = rs.getMetaData(); - VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator); - JdbcToArrowUtils.jdbcToArrowVectors(rs, root); - return root; - } catch (Exception exc) { - // just throw it out after logging - throw exc; - } finally { - if (rs != null) { - rs.close(); - } - if (stmt != null) { - stmt.close(); // test - } + try (Statement stmt = connection.createStatement()) { + return sqlToArrow(stmt.executeQuery(query), rootAllocator); } } /** - * This method returns ArrowDataFetcher Object that can be used to fetch and iterate on the data in the given - * database table. - * - * @param connection - Database connection Object - * @param tableName - Table name from which records will be fetched + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. * - * @return ArrowDataFetcher - Instance of ArrowDataFetcher which can be used to get Arrow Vector obejcts by calling its functionality + * @param resultSet + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws Exception */ - public static ArrowDataFetcher jdbcArrowDataFetcher(Connection connection, String tableName) { - assert connection != null: "JDBC conncetion object can not be null"; - assert tableName != null && tableName.length() > 0: "Table name can not be null or empty"; + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - return new ArrowDataFetcher(connection, tableName); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator); + rootAllocator.close(); + return root; } + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws Exception + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(rootAllocator, "Root Allocator object can not be null"); + + VectorSchemaRoot root = VectorSchemaRoot.create( + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData()), rootAllocator); + JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root); + return root; + } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 9b201a872bf..8b6a1c1be51 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -18,7 +18,24 @@ package org.apache.arrow.adapter.jdbc; -import org.apache.arrow.vector.*; + +import com.google.common.base.Preconditions; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -26,8 +43,18 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.nio.charset.Charset; -import java.sql.*; +import java.math.BigDecimal; + +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; import java.util.ArrayList; import java.util.List; @@ -80,9 +107,8 @@ public class JdbcToArrowUtils { */ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { - assert rsmd != null; + Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); -// ImmutableList.Builder fields = ImmutableList.builder(); List fields = new ArrayList<>(); int columnCount = rsmd.getColumnCount(); for (int i = 1; i <= columnCount; i++) { @@ -132,7 +158,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); break; case Types.TIMESTAMP: - // timezone is null + // TODO Need to handle timezone fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), null)); break; case Types.BINARY: @@ -141,7 +167,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); break; case Types.ARRAY: - // not handled + // TODO Need to handle this type // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: @@ -152,7 +178,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti break; default: - // no-op + // no-op, shouldn't get here break; } } @@ -180,10 +206,10 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * @param root Arrow {@link VectorSchemaRoot} object to populate * @throws Exception */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws Exception { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { - assert rs != null; - assert root != null; + Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); ResultSetMetaData rsmd = rs.getMetaData(); int columnCount = rsmd.getColumnCount(); @@ -200,46 +226,38 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw switch (rsmd.getColumnType(i)) { case Types.BOOLEAN: case Types.BIT: - BitVector bitVector = (BitVector) root.getVector(columnName); - bitVector.setSafe(rowCount, rs.getBoolean(i)? 1: 0); - bitVector.setValueCount(rowCount + 1); + updateVector((BitVector)root.getVector(columnName), + rs.getBoolean(i), rowCount); break; case Types.TINYINT: - TinyIntVector tinyIntVector = (TinyIntVector)root.getVector(columnName); - tinyIntVector.setSafe(rowCount, rs.getInt(i)); - tinyIntVector.setValueCount(rowCount + 1); + updateVector((TinyIntVector)root.getVector(columnName), + rs.getInt(i), rowCount); break; case Types.SMALLINT: - SmallIntVector smallIntVector = (SmallIntVector)root.getVector(columnName); - smallIntVector.setSafe(rowCount, rs.getInt(i)); - smallIntVector.setValueCount(rowCount + 1); + updateVector((SmallIntVector)root.getVector(columnName), + rs.getInt(i), rowCount); break; case Types.INTEGER: - IntVector intVector = (IntVector)root.getVector(columnName); - intVector.setSafe(rowCount, rs.getInt(i)); - intVector.setValueCount(rowCount + 1); + updateVector((IntVector)root.getVector(columnName), + rs.getInt(i), rowCount); break; case Types.BIGINT: - BigIntVector bigIntVector = (BigIntVector)root.getVector(columnName); - bigIntVector.setSafe(rowCount, rs.getInt(i)); - bigIntVector.setValueCount(rowCount + 1); + updateVector((BigIntVector)root.getVector(columnName), + rs.getInt(i), rowCount); break; case Types.NUMERIC: case Types.DECIMAL: - DecimalVector decimalVector = (DecimalVector)root.getVector(columnName); - decimalVector.setSafe(rowCount, rs.getBigDecimal(i)); - decimalVector.setValueCount(rowCount + 1); + updateVector((DecimalVector)root.getVector(columnName), + rs.getBigDecimal(i), rowCount); break; case Types.REAL: case Types.FLOAT: - Float4Vector float4Vector = (Float4Vector)root.getVector(columnName); - float4Vector.setSafe(rowCount, rs.getFloat(i)); - float4Vector.setValueCount(rowCount + 1); + updateVector((Float4Vector)root.getVector(columnName), + rs.getFloat(i), rowCount); break; case Types.DOUBLE: - Float8Vector float8Vector = (Float8Vector)root.getVector(columnName); - float8Vector.setSafe(rowCount, rs.getDouble(i)); - float8Vector.setValueCount(rowCount + 1); + updateVector((Float8Vector)root.getVector(columnName), + rs.getDouble(i), rowCount); break; case Types.CHAR: case Types.NCHAR: @@ -247,92 +265,42 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.NVARCHAR: case Types.LONGVARCHAR: case Types.LONGNVARCHAR: - VarCharVector varcharVector = (VarCharVector)root.getVector(columnName); - String value = rs.getString(i) != null ? rs.getString(i) : ""; - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, value.length()); - varcharVector.setSafe(rowCount, value.getBytes(Charset.forName("UTF-8")), 0, value.length()); - varcharVector.setValueCount(rowCount + 1); + updateVector((VarCharVector)root.getVector(columnName), + rs.getString(i), rowCount); break; case Types.DATE: - Date date = rs.getDate(i); - DateMilliVector dateMilliVector = (DateMilliVector) root.getVector(columnName); - dateMilliVector.setValueCount(rowCount + 1); - if (date != null) { - dateMilliVector.setSafe(rowCount, rs.getDate(i).getTime()); - } else { - dateMilliVector.setNull(rowCount); - } + updateVector((DateMilliVector) root.getVector(columnName), + rs.getDate(i), rowCount); break; case Types.TIME: - Time time = rs.getTime(i); - TimeMilliVector timeMilliVector = (TimeMilliVector)root.getVector(columnName); - timeMilliVector.setValueCount(rowCount + 1); - if (time != null) { - timeMilliVector.setSafe(rowCount, (int) rs.getTime(i).getTime()); - } else { - timeMilliVector.setNull(rowCount); - } - + updateVector((TimeMilliVector) root.getVector(columnName), + rs.getTime(i), rowCount); break; case Types.TIMESTAMP: - // timezone is null - Timestamp timestamp = rs.getTimestamp(i); - TimeStampVector timeStampVector = (TimeStampVector)root.getVector(columnName); - timeStampVector.setValueCount(rowCount + 1); - if (timestamp != null) { - timeStampVector.setSafe(rowCount, timestamp.getTime()); - } else { - timeStampVector.setNull(rowCount); - } + updateVector((TimeStampVector)root.getVector(columnName), + rs.getTimestamp(i), rowCount); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: - VarBinaryVector varBinaryVector = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector.setValueCount(rowCount + 1); - byte[] bytes = rs.getBytes(i); - if (bytes != null) { - varBinaryVector.setIndexDefined(rowCount); - varBinaryVector.setValueLengthSafe(rowCount, bytes.length); - varBinaryVector.setSafe(rowCount, bytes); - } else { - varBinaryVector.setNull(rowCount); - } + updateVector((VarBinaryVector)root.getVector(columnName), + rs.getBytes(i), rowCount); break; case Types.ARRAY: - // not handled + // TODO Need to handle this type // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: - VarCharVector varcharVector1 = (VarCharVector)root.getVector(columnName); - varcharVector1.setValueCount(rowCount + 1); - Clob clob = rs.getClob(i); - if (clob != null) { - int length = (int) clob.length(); - varcharVector1.setIndexDefined(rowCount); - varcharVector1.setValueLengthSafe(rowCount, length); - varcharVector1.setSafe(rowCount, clob.getSubString(1, length).getBytes(), 0, length); - } else { - varcharVector1.setNull(rowCount); - } + updateVector((VarCharVector)root.getVector(columnName), + rs.getClob(i), rowCount); break; case Types.BLOB: - VarBinaryVector varBinaryVector1 = (VarBinaryVector)root.getVector(columnName);; - varBinaryVector1.setValueCount(rowCount + 1); - Blob blob = rs.getBlob(i); - if (blob != null) { - byte[] data = blob.getBytes(0, (int) blob.length()); - varBinaryVector1.setIndexDefined(rowCount); - varBinaryVector1.setValueLengthSafe(rowCount, (int) blob.length()); - varBinaryVector1.setSafe(rowCount, data); - } else { - varBinaryVector1.setNull(rowCount);} - + updateVector((VarBinaryVector)root.getVector(columnName), + rs.getBlob(i), rowCount); break; default: - // no-op + // no-op, shouldn't get here break; } } @@ -340,4 +308,124 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw } root.setRowCount(rowCount); } + + private static void updateVector(BitVector bitVector, boolean value, int rowCount) { + bitVector.setSafe(rowCount, value? 1: 0); + bitVector.setValueCount(rowCount + 1); + } + + private static void updateVector(TinyIntVector tinyIntVector, int value, int rowCount) { + tinyIntVector.setSafe(rowCount, value); + tinyIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(SmallIntVector smallIntVector, int value, int rowCount) { + smallIntVector.setSafe(rowCount, value); + smallIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(IntVector intVector, int value, int rowCount) { + intVector.setSafe(rowCount, value); + intVector.setValueCount(rowCount + 1); + } + + private static void updateVector(BigIntVector bigIntVector, int value, int rowCount) { + bigIntVector.setSafe(rowCount, value); + bigIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(DecimalVector decimalVector, BigDecimal value, int rowCount) { + decimalVector.setSafe(rowCount, value); + decimalVector.setValueCount(rowCount + 1); + } + + private static void updateVector(Float4Vector float4Vector, float value, int rowCount) { + float4Vector.setSafe(rowCount, value); + float4Vector.setValueCount(rowCount + 1); + } + + private static void updateVector(Float8Vector float8Vector, double value, int rowCount) { + float8Vector.setSafe(rowCount, value); + float8Vector.setValueCount(rowCount + 1); + } + + private static void updateVector(VarCharVector varcharVector, String value, int rowCount) { + if (value != null) { + varcharVector.setIndexDefined(rowCount); + varcharVector.setValueLengthSafe(rowCount, value.length()); + varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, value.length()); + varcharVector.setValueCount(rowCount + 1); + } + // TODO: not sure how to handle null string value ??? + } + + private static void updateVector(DateMilliVector dateMilliVector, Date date, int rowCount) { + //TODO: Need to handle Timezone + dateMilliVector.setValueCount(rowCount + 1); + if (date != null) { + dateMilliVector.setSafe(rowCount, date.getTime()); + } else { + dateMilliVector.setNull(rowCount); + } + } + + private static void updateVector(TimeMilliVector timeMilliVector, Time time, int rowCount) { + timeMilliVector.setValueCount(rowCount + 1); + if (time != null) { + timeMilliVector.setSafe(rowCount, (int) time.getTime()); + } else { + timeMilliVector.setNull(rowCount); + } + } + + private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, int rowCount) { + //TODO Need to handle timezone ??? + timeStampVector.setValueCount(rowCount + 1); + if (timestamp != null) { + timeStampVector.setSafe(rowCount, timestamp.getTime()); + } else { + timeStampVector.setNull(rowCount); + } + } + + private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, int rowCount) { + varBinaryVector.setValueCount(rowCount + 1); + if (bytes != null) { + varBinaryVector.setIndexDefined(rowCount); + varBinaryVector.setValueLengthSafe(rowCount, bytes.length); + varBinaryVector.setSafe(rowCount, bytes); + } else { + varBinaryVector.setNull(rowCount); + } + } + + private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { + varcharVector.setValueCount(rowCount + 1); + if (clob != null) { + int length = (int) clob.length(); + String value = clob.getSubString(1, length); + if (value != null) { + varcharVector.setIndexDefined(rowCount); + varcharVector.setValueLengthSafe(rowCount, length); + varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, length); + } else { + varcharVector.setNull(rowCount); + } + } else { + varcharVector.setNull(rowCount); + } + } + + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int rowCount) throws SQLException { + varBinaryVector.setValueCount(rowCount + 1); + if (blob != null) { + byte[] data = blob.getBytes(0, (int) blob.length()); + varBinaryVector.setIndexDefined(rowCount); + varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); + varBinaryVector.setSafe(rowCount, data); + } else { + varBinaryVector.setNull(rowCount); + } + + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 2d3a328de92..5c409bbaa8a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -19,7 +19,7 @@ package org.apache.arrow.adapter.jdbc; /** - * + * POJO to handle the YAML data from the test YAML file. */ public class Table { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java deleted file mode 100644 index b7fca36818f..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/ArrowDataFetcherTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.adapter.jdbc.h2; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.arrow.adapter.jdbc.*; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.Connection; -import java.sql.DriverManager; - -import static org.junit.Assert.*; - -/** - * Test class for {@link ArrowDataFetcher}. - */ -public class ArrowDataFetcherTest extends AbstractJdbcToArrowTest { - - private Connection conn = null; - private ObjectMapper mapper = null; - - @Before - public void setUp() throws Exception { - String url = "jdbc:h2:mem:ArrowDataFetcherTest"; - String driver = "org.h2.Driver"; - - mapper = new ObjectMapper(new YAMLFactory()); - - Class.forName(driver); - - conn = DriverManager.getConnection(url); - } - - @After - public void destroy() throws Exception { - if (conn != null) { - conn.close(); - conn = null; - } - } - - @Test - public void commaSeparatedQueryColumnsTest() { - try { - ArrowDataFetcher.commaSeparatedQueryColumns(null); - } catch (AssertionError error) { - assertTrue(true); - } - assertEquals(" one ", ArrowDataFetcher.commaSeparatedQueryColumns("one")); - assertEquals(" one, two ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two")); - assertEquals(" one, two, three ", ArrowDataFetcher.commaSeparatedQueryColumns("one", "two", "three")); - } - - @Test - public void arrowFetcherAllColumnsLimitOffsetTest() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), - Table.class); - - try { - createTestData(conn, table); - - ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(conn, "table1"); - - VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10); - - int[] values = { - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 - }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 10, values); - - root = arrowDataFetcher.fetch(5, 5); - - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 5, values); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - - } - - @Test - public void arrowFetcherSingleColumnLimitOffsetTest() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), - Table.class); - - try { - createTestData(conn, table); - - ArrowDataFetcher arrowDataFetcher = JdbcToArrow.jdbcArrowDataFetcher(conn, "table1"); - - VectorSchemaRoot root = arrowDataFetcher.fetch(0, 10, "int_field1"); - - int[] values = { - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 - }; - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 10, values); - - root = arrowDataFetcher.fetch(5, 5); - - JdbcToArrowTestHelper.assertIntVectorValues(root.getVector("INT_FIELD1"), 5, values); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - - } - - -} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 89558d961f0..288c4c8b1bf 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -23,6 +23,7 @@ import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; @@ -43,6 +44,7 @@ public class JdbcToArrowTest extends AbstractJdbcToArrowTest { private Connection conn = null; private ObjectMapper mapper = null; + @Before public void setUp() throws Exception { String url = "jdbc:h2:mem:JdbcToArrowTest"; @@ -71,10 +73,11 @@ public void sqlToArrowTestInt() throws Exception { this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), Table.class); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] values = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, @@ -82,7 +85,7 @@ public void sqlToArrowTestInt() throws Exception { assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); } catch (Exception e) { - e.printStackTrace(); + throw e; } finally { deleteTestData(conn, table); } @@ -96,11 +99,12 @@ public void sqlToArrowTestBool() throws Exception { mapper.readValue( this.getClass().getClassLoader().getResourceAsStream("h2/test1_bool_h2.yml"), Table.class); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 @@ -122,11 +126,11 @@ public void sqlToArrowTestTinyInt() throws Exception { mapper.readValue( this.getClass().getClassLoader().getResourceAsStream("h2/test1_tinyint_h2.yml"), Table.class); - + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 @@ -148,11 +152,11 @@ public void sqlToArrowTestSmallInt() throws Exception { mapper.readValue( this.getClass().getClassLoader().getResourceAsStream("h2/test1_smallint_h2.yml"), Table.class); - + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 @@ -174,11 +178,11 @@ public void sqlToArrowTestBigInt() throws Exception { mapper.readValue( this.getClass().getClassLoader().getResourceAsStream("h2/test1_bigint_h2.yml"), Table.class); - + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, @@ -202,10 +206,11 @@ public void sqlToArrowTestAllDataTypes() throws Exception { this.getClass().getClassLoader().getResourceAsStream("h2/test1_all_datatypes_h2.yml"), Table.class); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); try { createTestData(conn, table); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery()); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 From 8b233967232a3abbc218f8dd2779d5d99b62db38 Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 6 Apr 2018 10:12:38 -0400 Subject: [PATCH 068/129] files committed after making changes for review comment implementation --- .../arrow/adapter/jdbc/JdbcToArrow.java | 11 +-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 72 ++++++++++--------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index e375a0264a8..37eb9e14359 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -71,12 +72,12 @@ public class JdbcToArrow { * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException, IOException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); - + try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), rootAllocator); + return sqlToArrow(stmt.executeQuery(query), rootAllocator); } } @@ -87,7 +88,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, R * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws Exception */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); @@ -103,7 +104,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws Exception */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(rootAllocator, "Root Allocator object can not be null"); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 8b6a1c1be51..20ec325424b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -20,6 +20,9 @@ import com.google.common.base.Preconditions; +import com.google.common.io.ByteStreams; +import com.google.common.io.CharStreams; + import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -43,11 +46,12 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.math.BigDecimal; - +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.sql.Blob; -import java.sql.Clob; import java.sql.Date; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -61,7 +65,6 @@ import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; - /** * Class that does most of the work to convert JDBC ResultSet data into Arrow columnar format Vector objects. * @@ -168,7 +171,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti break; case Types.ARRAY: // TODO Need to handle this type -// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); @@ -206,7 +209,7 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * @param root Arrow {@link VectorSchemaRoot} object to populate * @throws Exception */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException, IOException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); @@ -219,7 +222,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int rowCount = 0; while (rs.next()) { // for each column get the value based on the type - // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); @@ -243,7 +245,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.BIGINT: updateVector((BigIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getLong(i), rowCount); break; case Types.NUMERIC: case Types.DECIMAL: @@ -288,15 +290,15 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.ARRAY: // TODO Need to handle this type -// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: - updateVector((VarCharVector)root.getVector(columnName), - rs.getClob(i), rowCount); + updateVector((VarCharVector)root.getVector(columnName), + rs.getCharacterStream(i), rowCount); break; case Types.BLOB: - updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBlob(i), rowCount); + updateVector((VarBinaryVector)root.getVector(columnName), + rs.getBinaryStream(i), rowCount); break; default: @@ -329,7 +331,7 @@ private static void updateVector(IntVector intVector, int value, int rowCount) intVector.setValueCount(rowCount + 1); } - private static void updateVector(BigIntVector bigIntVector, int value, int rowCount) { + private static void updateVector(BigIntVector bigIntVector, long value, int rowCount) { bigIntVector.setSafe(rowCount, value); bigIntVector.setValueCount(rowCount + 1); } @@ -398,34 +400,38 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, varBinaryVector.setNull(rowCount); } } - - private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { + + private static void updateVector(VarCharVector varcharVector, Reader clobReader, int rowCount) throws SQLException, IOException { + Preconditions.checkNotNull(clobReader, "Reader object for Clob can't be null"); + + ByteBuffer clobBuff = ByteBuffer.wrap(CharStreams.toString(clobReader).getBytes()); + clobReader.close(); varcharVector.setValueCount(rowCount + 1); - if (clob != null) { - int length = (int) clob.length(); - String value = clob.getSubString(1, length); - if (value != null) { - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, length); - varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, length); - } else { - varcharVector.setNull(rowCount); - } + if (clobBuff != null) { + int length = clobBuff.capacity(); + varcharVector.setIndexDefined(rowCount); + varcharVector.setValueLengthSafe(rowCount, length); + varcharVector.setSafe(rowCount, clobBuff, 0, length); } else { varcharVector.setNull(rowCount); } } - private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int rowCount) throws SQLException { - varBinaryVector.setValueCount(rowCount + 1); - if (blob != null) { - byte[] data = blob.getBytes(0, (int) blob.length()); + private static void updateVector(VarBinaryVector varBinaryVector, InputStream blobStream, int rowCount) throws SQLException, IOException { + Preconditions.checkNotNull(blobStream, "InputStream object for Blob can't be null"); + + ByteBuffer blobBuff = ByteBuffer.wrap(ByteStreams.toByteArray(blobStream)); + blobStream.close(); + + varBinaryVector.setValueCount(rowCount + 1); + if (blobBuff != null) { + int length = blobBuff.capacity(); varBinaryVector.setIndexDefined(rowCount); - varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); - varBinaryVector.setSafe(rowCount, data); + varBinaryVector.setValueLengthSafe(rowCount, length); + varBinaryVector.setSafe(rowCount, blobBuff, 0, length); } else { varBinaryVector.setNull(rowCount); } - } + } From e55cb48798ad3494603f510be43e36ccc7f0cac9 Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 6 Apr 2018 10:13:23 -0400 Subject: [PATCH 069/129] files committed after making changes for review comment implementation --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 148 ++++++------------ 1 file changed, 49 insertions(+), 99 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index ef1fa39591e..090dea9f396 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -19,6 +19,7 @@ package org.apache.arrow.adapter.jdbc; import java.math.BigDecimal; +import java.util.Arrays; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -35,190 +36,139 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; -import static org.junit.Assert.*; - - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** - * This is a Helper class which has functionalities to read and assert the values from teh given FieldVector object + * This is a Helper class which has functionalities to read and assert the values from the given FieldVector object * */ public class JdbcToArrowTestHelper { - public static boolean assertIntVectorValues(FieldVector fx, int rowCount, int[] values) { + public static void assertIntVectorValues(FieldVector fx, int rowCount, int[] values) { IntVector intVector = ((IntVector) fx); - assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - if(!intVector.isNull(j)) { - assertEquals(values[j], intVector.get(j)); - } + //assertNotNull(intVector.get(j)); + assertEquals(values[j], intVector.get(j)); } - return true; } - public static boolean assertBitBooleanVectorValues(FieldVector fx, int rowCount, int[] values){ + public static void assertBitBooleanVectorValues(FieldVector fx, int rowCount, int[] values){ BitVector bitVector = ((BitVector)fx); assertEquals(rowCount, bitVector.getValueCount()); + for(int j = 0; j < bitVector.getValueCount(); j++){ - if(!bitVector.isNull(j)) { - assertEquals(values[j], bitVector.get(j)); - } + assertEquals(values[j], bitVector.get(j)); } - return true; } - public static boolean assertTinyIntVectorValues(FieldVector fx, int rowCount, int[] values){ + public static void assertTinyIntVectorValues(FieldVector fx, int rowCount, int[] values){ TinyIntVector tinyIntVector = ((TinyIntVector)fx); - assertEquals(rowCount, tinyIntVector.getValueCount()); for(int j = 0; j < tinyIntVector.getValueCount(); j++){ - if(!tinyIntVector.isNull(j)) { - assertEquals(values[j], tinyIntVector.get(j)); - } + assertEquals(values[j], tinyIntVector.get(j)); } - return true; } - public static boolean assertSmallIntVectorValues(FieldVector fx, int rowCount, int[] values){ + public static void assertSmallIntVectorValues(FieldVector fx, int rowCount, int[] values){ SmallIntVector smallIntVector = ((SmallIntVector)fx); - assertEquals(rowCount, smallIntVector.getValueCount()); for(int j = 0; j < smallIntVector.getValueCount(); j++){ - if(!smallIntVector.isNull(j)){ - assertEquals(values[j], smallIntVector.get(j)); - } + assertEquals(values[j], smallIntVector.get(j)); } - - return true; } - public static boolean assertBigIntVectorValues(FieldVector fx, int rowCount, int[] values){ + public static void assertBigIntVectorValues(FieldVector fx, int rowCount, int[] values){ BigIntVector bigIntVector = ((BigIntVector)fx); - assertEquals(rowCount, bigIntVector.getValueCount()); for(int j = 0; j < bigIntVector.getValueCount(); j++){ - if(!bigIntVector.isNull(j)) { - assertEquals(values[j], bigIntVector.get(j)); - } + assertEquals(values[j], bigIntVector.get(j)); } - - return true; } - public static boolean assertDecimalVectorValues(FieldVector fx, int rowCount, BigDecimal[] values){ + public static void assertDecimalVectorValues(FieldVector fx, int rowCount, BigDecimal[] values){ DecimalVector decimalVector = ((DecimalVector)fx); - assertEquals(rowCount, decimalVector.getValueCount()); for(int j = 0; j < decimalVector.getValueCount(); j++){ - if(!decimalVector.isNull(j)){ - assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); - } + assertNotNull(decimalVector.getObject(j)); + assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); } - - return true; } - public static boolean assertFloat8VectorValues(FieldVector fx, int rowCount, double[] values){ + public static void assertFloat8VectorValues(FieldVector fx, int rowCount, double[] values){ Float8Vector float8Vector = ((Float8Vector)fx); - assertEquals(rowCount, float8Vector.getValueCount()); for(int j = 0; j < float8Vector.getValueCount(); j++){ - if(!float8Vector.isNull(j)) { - assertEquals(values[j], float8Vector.get(j), 0.01); - } + assertEquals(values[j], float8Vector.get(j), 0.01); } - - return true; } - public static boolean assertFloat4VectorValues(FieldVector fx, int rowCount, float[] values){ + public static void assertFloat4VectorValues(FieldVector fx, int rowCount, float[] values){ Float4Vector float4Vector = ((Float4Vector)fx); - assertEquals(rowCount, float4Vector.getValueCount()); for(int j = 0; j < float4Vector.getValueCount(); j++){ - if(!float4Vector.isNull(j)){ - assertEquals(values[j], float4Vector.get(j), 0.01); - } + assertEquals(values[j], float4Vector.get(j), 0.01); } - - return true; } - public static boolean assertTimeVectorValues(FieldVector fx, int rowCount, long[] values){ + public static void assertTimeVectorValues(FieldVector fx, int rowCount, long[] values){ TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); - assertEquals(rowCount, timeMilliVector.getValueCount()); for(int j = 0; j < timeMilliVector.getValueCount(); j++){ - if(!timeMilliVector.isNull(j)){ assertEquals(values[j], timeMilliVector.get(j)); - } } - - return true; } - public static boolean assertDateVectorValues(FieldVector fx, int rowCount, long[] values){ + public static void assertDateVectorValues(FieldVector fx, int rowCount, long[] values){ DateMilliVector dateMilliVector = ((DateMilliVector)fx); - assertEquals(rowCount, dateMilliVector.getValueCount()); for(int j = 0; j < dateMilliVector.getValueCount(); j++){ - if(!dateMilliVector.isNull(j)){ - assertEquals(values[j], dateMilliVector.get(j)); - } + assertEquals(values[j], dateMilliVector.get(j)); } - - return true; } - public static boolean assertTimeStampVectorValues(FieldVector fx, int rowCount, long[] values){ + public static void assertTimeStampVectorValues(FieldVector fx, int rowCount, long[] values){ TimeStampVector timeStampVector = ((TimeStampVector)fx); - assertEquals(rowCount, timeStampVector.getValueCount()); for(int j = 0; j < timeStampVector.getValueCount(); j++){ - if(!timeStampVector.isNull(j)){ - assertEquals(values[j], timeStampVector.get(j)); - } + assertEquals(values[j], timeStampVector.get(j)); } - - return true; } - public static boolean assertVarBinaryVectorValues(FieldVector fx, int rowCount, byte[][] values){ - VarBinaryVector varBinaryVector =((VarBinaryVector) fx); - - assertEquals(rowCount, varBinaryVector.getValueCount()); - - for(int j = 0; j < varBinaryVector.getValueCount(); j++){ - if(!varBinaryVector.isNull(j)){ - assertEquals(hashArray(values[j]), hashArray(varBinaryVector.get(j))); - } + public static void assertVarBinaryVectorValues (FieldVector fx, int rowCount, byte[][] values) { + try { + VarBinaryVector varBinaryVector =((VarBinaryVector) fx); + assertEquals(rowCount, varBinaryVector.getValueCount()); + + for(int j = 0; j < varBinaryVector.getValueCount(); j++){ + assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varBinaryVector.get(j))); + } + } catch (AssertionError ae) { + ae.printStackTrace(); } - - return true; } - public static boolean assertVarcharVectorValues(FieldVector fx, int rowCount, byte[][] values) { - VarCharVector varCharVector = ((VarCharVector)fx); - - assertEquals(rowCount, varCharVector.getValueCount()); - - for(int j = 0; j < varCharVector.getValueCount(); j++){ - if(!varCharVector.isNull(j)){ - assertEquals(hashArray(values[j]), hashArray(varCharVector.get(j))); - } - } - - return true; + public static void assertVarcharVectorValues(FieldVector fx, int rowCount, byte[][] values) { + try { + VarCharVector varCharVector = ((VarCharVector)fx); + assertEquals(rowCount, varCharVector.getValueCount()); + + for(int j = 0; j < varCharVector.getValueCount(); j++){ + assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varCharVector.get(j))); + } + } catch (AssertionError ae) { + ae.printStackTrace(); + } } public static long hashArray(byte[] data){ From fbd84fbc39ce4065c5c04c84bfd74e801b5d842d Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 10 Apr 2018 09:02:25 -0400 Subject: [PATCH 070/129] File cimmitted after making changes for Blob and Clobe issue --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 20ec325424b..83d66ffabf0 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -20,9 +20,6 @@ import com.google.common.base.Preconditions; -import com.google.common.io.ByteStreams; -import com.google.common.io.CharStreams; - import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -45,7 +42,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; - +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -61,10 +58,10 @@ import java.sql.Types; import java.util.ArrayList; import java.util.List; - import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; + /** * Class that does most of the work to convert JDBC ResultSet data into Arrow columnar format Vector objects. * @@ -403,15 +400,24 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, private static void updateVector(VarCharVector varcharVector, Reader clobReader, int rowCount) throws SQLException, IOException { Preconditions.checkNotNull(clobReader, "Reader object for Clob can't be null"); - - ByteBuffer clobBuff = ByteBuffer.wrap(CharStreams.toString(clobReader).getBytes()); + + char[] charArr = new char[1024]; + StringBuilder clobBuilder = new StringBuilder(); + int charsRead; + + while ((charsRead = clobReader.read(charArr, 0, charArr.length)) != -1) { + clobBuilder.append(charArr, 0, charsRead); + } + + ByteBuffer clobBuffer = ByteBuffer.wrap(clobBuilder.toString().getBytes()); clobReader.close(); + varcharVector.setValueCount(rowCount + 1); - if (clobBuff != null) { - int length = clobBuff.capacity(); + if (clobBuffer != null) { + int length = clobBuffer.capacity(); varcharVector.setIndexDefined(rowCount); varcharVector.setValueLengthSafe(rowCount, length); - varcharVector.setSafe(rowCount, clobBuff, 0, length); + varcharVector.setSafe(rowCount, clobBuffer, 0, length); } else { varcharVector.setNull(rowCount); } @@ -419,10 +425,20 @@ private static void updateVector(VarCharVector varcharVector, Reader clobReader, private static void updateVector(VarBinaryVector varBinaryVector, InputStream blobStream, int rowCount) throws SQLException, IOException { Preconditions.checkNotNull(blobStream, "InputStream object for Blob can't be null"); - - ByteBuffer blobBuff = ByteBuffer.wrap(ByteStreams.toByteArray(blobStream)); + + ByteArrayOutputStream baoStream = new ByteArrayOutputStream(); + byte [] blobToByte = new byte [1024]; + + while(true) { + int length = blobStream.read(blobToByte); + if(length < 0) break; + baoStream.write(blobToByte, 0, length); + } + + ByteBuffer blobBuff = ByteBuffer.wrap(baoStream.toByteArray()); blobStream.close(); - + baoStream.close(); + varBinaryVector.setValueCount(rowCount + 1); if (blobBuff != null) { int length = blobBuff.capacity(); From 2c56d59de609671cda4ce2a0959eeaadc1149027 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 10 Apr 2018 09:04:26 -0400 Subject: [PATCH 071/129] File committed for Blob and Clob issue and also for moving typecasting into Test (caller) class --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 59 +++++-------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 090dea9f396..818fa0365b6 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -25,7 +25,6 @@ import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -44,18 +43,15 @@ */ public class JdbcToArrowTestHelper { - public static void assertIntVectorValues(FieldVector fx, int rowCount, int[] values) { - IntVector intVector = ((IntVector) fx); + public static void assertIntVectorValues(IntVector intVector, int rowCount, int [] values) { assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - //assertNotNull(intVector.get(j)); - assertEquals(values[j], intVector.get(j)); + assertEquals((int)values[j], intVector.get(j)); } } - public static void assertBitBooleanVectorValues(FieldVector fx, int rowCount, int[] values){ - BitVector bitVector = ((BitVector)fx); + public static void assertBitBooleanVectorValues(BitVector bitVector, int rowCount, int[] values){ assertEquals(rowCount, bitVector.getValueCount()); for(int j = 0; j < bitVector.getValueCount(); j++){ @@ -63,8 +59,7 @@ public static void assertBitBooleanVectorValues(FieldVector fx, int rowCount, in } } - public static void assertTinyIntVectorValues(FieldVector fx, int rowCount, int[] values){ - TinyIntVector tinyIntVector = ((TinyIntVector)fx); + public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, int[] values){ assertEquals(rowCount, tinyIntVector.getValueCount()); for(int j = 0; j < tinyIntVector.getValueCount(); j++){ @@ -72,8 +67,7 @@ public static void assertTinyIntVectorValues(FieldVector fx, int rowCount, int[] } } - public static void assertSmallIntVectorValues(FieldVector fx, int rowCount, int[] values){ - SmallIntVector smallIntVector = ((SmallIntVector)fx); + public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, int[] values){ assertEquals(rowCount, smallIntVector.getValueCount()); for(int j = 0; j < smallIntVector.getValueCount(); j++){ @@ -81,8 +75,7 @@ public static void assertSmallIntVectorValues(FieldVector fx, int rowCount, int[ } } - public static void assertBigIntVectorValues(FieldVector fx, int rowCount, int[] values){ - BigIntVector bigIntVector = ((BigIntVector)fx); + public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, int[] values){ assertEquals(rowCount, bigIntVector.getValueCount()); for(int j = 0; j < bigIntVector.getValueCount(); j++){ @@ -90,8 +83,7 @@ public static void assertBigIntVectorValues(FieldVector fx, int rowCount, int[] } } - public static void assertDecimalVectorValues(FieldVector fx, int rowCount, BigDecimal[] values){ - DecimalVector decimalVector = ((DecimalVector)fx); + public static void assertDecimalVectorValues(DecimalVector decimalVector, int rowCount, BigDecimal[] values){ assertEquals(rowCount, decimalVector.getValueCount()); for(int j = 0; j < decimalVector.getValueCount(); j++){ @@ -100,8 +92,7 @@ public static void assertDecimalVectorValues(FieldVector fx, int rowCount, BigDe } } - public static void assertFloat8VectorValues(FieldVector fx, int rowCount, double[] values){ - Float8Vector float8Vector = ((Float8Vector)fx); + public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, double[] values){ assertEquals(rowCount, float8Vector.getValueCount()); for(int j = 0; j < float8Vector.getValueCount(); j++){ @@ -109,8 +100,7 @@ public static void assertFloat8VectorValues(FieldVector fx, int rowCount, double } } - public static void assertFloat4VectorValues(FieldVector fx, int rowCount, float[] values){ - Float4Vector float4Vector = ((Float4Vector)fx); + public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, float[] values){ assertEquals(rowCount, float4Vector.getValueCount()); for(int j = 0; j < float4Vector.getValueCount(); j++){ @@ -118,8 +108,7 @@ public static void assertFloat4VectorValues(FieldVector fx, int rowCount, float[ } } - public static void assertTimeVectorValues(FieldVector fx, int rowCount, long[] values){ - TimeMilliVector timeMilliVector = ((TimeMilliVector)fx); + public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, long[] values){ assertEquals(rowCount, timeMilliVector.getValueCount()); for(int j = 0; j < timeMilliVector.getValueCount(); j++){ @@ -127,8 +116,7 @@ public static void assertTimeVectorValues(FieldVector fx, int rowCount, long[] v } } - public static void assertDateVectorValues(FieldVector fx, int rowCount, long[] values){ - DateMilliVector dateMilliVector = ((DateMilliVector)fx); + public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, long[] values){ assertEquals(rowCount, dateMilliVector.getValueCount()); for(int j = 0; j < dateMilliVector.getValueCount(); j++){ @@ -136,8 +124,7 @@ public static void assertDateVectorValues(FieldVector fx, int rowCount, long[] v } } - public static void assertTimeStampVectorValues(FieldVector fx, int rowCount, long[] values){ - TimeStampVector timeStampVector = ((TimeStampVector)fx); + public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, long[] values){ assertEquals(rowCount, timeStampVector.getValueCount()); for(int j = 0; j < timeStampVector.getValueCount(); j++){ @@ -145,9 +132,8 @@ public static void assertTimeStampVectorValues(FieldVector fx, int rowCount, lon } } - public static void assertVarBinaryVectorValues (FieldVector fx, int rowCount, byte[][] values) { + public static void assertVarBinaryVectorValues (VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { try { - VarBinaryVector varBinaryVector =((VarBinaryVector) fx); assertEquals(rowCount, varBinaryVector.getValueCount()); for(int j = 0; j < varBinaryVector.getValueCount(); j++){ @@ -158,9 +144,8 @@ public static void assertVarBinaryVectorValues (FieldVector fx, int rowCount, by } } - public static void assertVarcharVectorValues(FieldVector fx, int rowCount, byte[][] values) { + public static void assertVarcharVectorValues(VarCharVector varCharVector, int rowCount, byte[][] values) { try { - VarCharVector varCharVector = ((VarCharVector)fx); assertEquals(rowCount, varCharVector.getValueCount()); for(int j = 0; j < varCharVector.getValueCount(); j++){ @@ -171,22 +156,6 @@ public static void assertVarcharVectorValues(FieldVector fx, int rowCount, byte[ } } - public static long hashArray(byte[] data){ - long ret = 0; - for(int i = 0; i < data.length;i++) - ret+=data[i]; - return ret; - } - - public static String firstX(byte[] data, int items){ - int toProcess = Math.min(items, data.length); - StringBuilder sb = new StringBuilder(); - for(int i = 0; i < toProcess; i++) { - sb.append(String.format("0x%02x", data[i])+ " "); - } - return sb.toString(); - } - public static byte[] hexStringToByteArray(String s) { int len = s.length(); byte[] data = new byte[len / 2]; From b8dce969ed338d0b1b24bc99cb621f4115193b19 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 10 Apr 2018 09:04:55 -0400 Subject: [PATCH 072/129] File committed for Blob and Clob issue and also for moving typecasting into Test (caller) class --- .../adapter/jdbc/h2/JdbcToArrowTest.java | 128 ++++++++++++++---- 1 file changed, 103 insertions(+), 25 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 288c4c8b1bf..fd98ec61f7a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -25,6 +25,20 @@ import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -32,7 +46,6 @@ import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.util.Properties; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; @@ -82,7 +95,7 @@ public void sqlToArrowTestInt() throws Exception { int[] values = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, }; - assertIntVectorValues(root.getVector("INT_FIELD1"), 15, values); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, values); } catch (Exception e) { throw e; @@ -109,7 +122,7 @@ public void sqlToArrowTestBool() throws Exception { int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); } catch (Exception e) { e.printStackTrace(); @@ -135,7 +148,7 @@ public void sqlToArrowTestTinyInt() throws Exception { int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); } catch (Exception e) { e.printStackTrace(); @@ -161,7 +174,7 @@ public void sqlToArrowTestSmallInt() throws Exception { int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); } catch (Exception e) { e.printStackTrace(); @@ -188,7 +201,47 @@ public void sqlToArrowTestBigInt() throws Exception { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(conn, table); + } + + } + + @Test + public void sqlToArrowTestBlob() throws Exception { + + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("h2/test1_blob_h2.yml"), + Table.class); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + try { + createTestData(conn, table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); + + byte[][] bytes = { + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), + hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") + }; + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); } catch (Exception e) { e.printStackTrace(); @@ -198,6 +251,31 @@ public void sqlToArrowTestBigInt() throws Exception { } + @Test + public void sqlToArrowTestClobs() throws Exception { + Table table = + mapper.readValue( + this.getClass().getClassLoader().getResourceAsStream("h2/test1_clob_h2.yml"), + Table.class); + + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + try { + createTestData(conn, table); + + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); + + byte[] strb = "some text that needs to be converted to clob".getBytes(); + byte[][] varchars = { + strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb + }; + assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTestData(conn, table); + } + } + @Test public void sqlToArrowTestAllDataTypes() throws Exception { @@ -215,28 +293,28 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - assertIntVectorValues(root.getVector("INT_FIELD1"), 15, ints); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, ints); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - assertBitBooleanVectorValues(root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - assertTinyIntVectorValues(root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - assertSmallIntVectorValues(root.getVector("SMALLINT_FIELD4"), 15, smallints); + assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - assertBigIntVectorValues(root.getVector("BIGINT_FIELD5"), 15, bigints); + assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); BigDecimal[] bigdecimals = { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), @@ -245,39 +323,39 @@ public void sqlToArrowTestAllDataTypes() throws Exception { new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) }; - assertDecimalVectorValues(root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); + assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); double[] doubles = { 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 }; - assertFloat8VectorValues(root.getVector("DOUBLE_FIELD7"), 15, doubles); + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 15, doubles); float[] reals = { 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f }; - assertFloat4VectorValues(root.getVector("REAL_FIELD8"), 15, reals); + assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), 15, reals); long[] times = { 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000 }; - assertTimeVectorValues(root.getVector("TIME_FIELD9"), 15, times); + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 15, times); long[] dates = { - 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, - 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l + 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, + 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l }; - assertDateVectorValues(root.getVector("DATE_FIELD10"), 15, dates); + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), 15, dates); long[] timestamps = { 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l }; - assertTimeStampVectorValues(root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); byte[][] bytes = { hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), @@ -296,29 +374,29 @@ public void sqlToArrowTestAllDataTypes() throws Exception { hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") }; - assertVarBinaryVectorValues(root.getVector("BINARY_FIELD12"), 15, bytes); + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 15, bytes); byte[] strb = "some text that needs to be converted to varchar".getBytes(); byte[][] varchars = { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - assertVarcharVectorValues(root.getVector("VARCHAR_FIELD13"), 15, varchars); + assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 15, varchars); - assertVarBinaryVectorValues(root.getVector("BLOB_FIELD14"), 15, bytes); + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); strb = "some text that needs to be converted to clob".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - assertVarcharVectorValues(root.getVector("CLOB_FIELD15"), 15, varchars); + assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); strb = "some char text".getBytes(); varchars = new byte[][] { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - assertVarcharVectorValues(root.getVector("CHAR_FIELD16"), 15, varchars); + assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), 15, varchars); - assertBitBooleanVectorValues(root.getVector("BIT_FIELD17"), 15, bools); + assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), 15, bools); } catch (Exception e) { e.printStackTrace(); } finally { From 85a23548600dd117eae8a9850d4ebbbdb23dc4f9 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 10 Apr 2018 14:24:15 -0400 Subject: [PATCH 073/129] added new test class for INT values --- .../arrow/adapter/jdbc/h2/TestIntValues.java | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java new file mode 100644 index 00000000000..e07f6d83769 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Test; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; + +@RunWith(Parameterized.class) +public class TestIntValues { + private static Connection conn = null; + private static Table table; + + // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor + public int [] intValues; + + // This is the constructor which will add data to @Parameter annotated variable + public TestIntValues (IntegerValues integerValues) { + super(); + this.intValues = integerValues.intValues; + } + + // This is the method which is annotated for injecting values into the test class constructor + @Parameters + public static Collection getTestData() throws Exception { + // Calling setUp() method here in place of using it with @Before annotation because this method is called before @Before annotated + // method and we need Table object and also Connection object Because for this class creating table and populating it + // with data using YML file only which should be done automatically as per the comment so this will be change and it place of + // this we can call method to just create Table object (for ex - getTable(ymlFileNamePath)) + setUp(); + + final List valueList = new ArrayList<>(); + + Arrays.stream(table.getData()).parallel().forEach(value -> { + valueList.add(Integer.parseInt(value)); + }); + return Arrays.asList(new Object[][]{ + {new IntegerValues(valueList.stream().mapToInt(Integer::intValue).toArray())}}); + } + + // This is the test method after making changes as per comment + @Test + public void sqlToArrowTestInt() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // @Before + public static void setUp() throws Exception { + //try { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + + Class.forName(driver); + conn = DriverManager.getConnection(url); + + table = getTable ("h2/test1_int_h2.yml"); + createTestData(conn, table); + //} catch () + } + + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + protected static Table getTable (String ymlFilePath) { + Table table = null; + try { + table = new ObjectMapper(new YAMLFactory()).readValue( + TestIntValues.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } catch (JsonMappingException jme) { + jme.printStackTrace(); + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + return table; + } + + protected static void createTestData(Connection conn, Table table) { + + Statement stmt = null; + try { + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getInserts()) { + stmt.executeUpdate(insert); + } + + } catch (SQLException e) { + e.printStackTrace(); + } finally { + //JdbcToArrowUtils.closeStatement(stmt); + } + } + +} + +class IntegerValues { + + public int [] intValues; + + public IntegerValues(int [] intValues) { + this.intValues = intValues; + } +} From e1ed84dd2d330ef6ad92f1f89caaf383e5553aa1 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 10 Apr 2018 14:31:57 -0400 Subject: [PATCH 074/129] Committed this class for new test class --- .../java/org/apache/arrow/adapter/jdbc/Table.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 5c409bbaa8a..2888435d63d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -25,6 +25,7 @@ public class Table { private String name; private String create; + private String [] inserts; private String[] data; private String query; private String drop; @@ -32,7 +33,15 @@ public class Table { public Table() { } - public String getName() { + public String[] getInserts() { + return inserts; + } + + public void setInserts(String[] inserts) { + this.inserts = inserts; + } + + public String getName() { return name; } From f7571640b124b12443ccfc1bfa380b7efe3ce55d Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 11 Apr 2018 02:48:47 -0400 Subject: [PATCH 075/129] files committed for reverting Blob and Clob related changes --- .../arrow/adapter/jdbc/JdbcToArrow.java | 8 +-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 72 +++++++------------ 2 files changed, 28 insertions(+), 52 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 37eb9e14359..7038e9a9b1e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; -import java.io.IOException; + import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -72,7 +72,7 @@ public class JdbcToArrow { * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException, IOException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); @@ -88,7 +88,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, R * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws Exception */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); @@ -104,7 +104,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws Exception */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException, IOException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(rootAllocator, "Root Allocator object can not be null"); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 83d66ffabf0..2745b7a728b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -42,13 +42,10 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.Clob; import java.sql.Date; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -206,7 +203,7 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * @param root Arrow {@link VectorSchemaRoot} object to populate * @throws Exception */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException, IOException { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); @@ -291,11 +288,11 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.CLOB: updateVector((VarCharVector)root.getVector(columnName), - rs.getCharacterStream(i), rowCount); + rs.getClob(i), rowCount); break; case Types.BLOB: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBinaryStream(i), rowCount); + rs.getBlob(i), rowCount); break; default: @@ -398,56 +395,35 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, } } - private static void updateVector(VarCharVector varcharVector, Reader clobReader, int rowCount) throws SQLException, IOException { - Preconditions.checkNotNull(clobReader, "Reader object for Clob can't be null"); - - char[] charArr = new char[1024]; - StringBuilder clobBuilder = new StringBuilder(); - int charsRead; - - while ((charsRead = clobReader.read(charArr, 0, charArr.length)) != -1) { - clobBuilder.append(charArr, 0, charsRead); - } - - ByteBuffer clobBuffer = ByteBuffer.wrap(clobBuilder.toString().getBytes()); - clobReader.close(); - + private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { varcharVector.setValueCount(rowCount + 1); - if (clobBuffer != null) { - int length = clobBuffer.capacity(); - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, length); - varcharVector.setSafe(rowCount, clobBuffer, 0, length); + if (clob != null) { + int length = (int) clob.length(); + String value = clob.getSubString(1, length); + if (value != null) { + varcharVector.setIndexDefined(rowCount); + varcharVector.setValueLengthSafe(rowCount, length); + varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, length); + } else { + varcharVector.setNull(rowCount); + } } else { varcharVector.setNull(rowCount); } } - private static void updateVector(VarBinaryVector varBinaryVector, InputStream blobStream, int rowCount) throws SQLException, IOException { - Preconditions.checkNotNull(blobStream, "InputStream object for Blob can't be null"); - - ByteArrayOutputStream baoStream = new ByteArrayOutputStream(); - byte [] blobToByte = new byte [1024]; - - while(true) { - int length = blobStream.read(blobToByte); - if(length < 0) break; - baoStream.write(blobToByte, 0, length); - } - - ByteBuffer blobBuff = ByteBuffer.wrap(baoStream.toByteArray()); - blobStream.close(); - baoStream.close(); - - varBinaryVector.setValueCount(rowCount + 1); - if (blobBuff != null) { - int length = blobBuff.capacity(); + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int rowCount) throws SQLException { + varBinaryVector.setValueCount(rowCount + 1); + if (blob != null) { + byte[] data = blob.getBytes(0, (int) blob.length()); varBinaryVector.setIndexDefined(rowCount); - varBinaryVector.setValueLengthSafe(rowCount, length); - varBinaryVector.setSafe(rowCount, blobBuff, 0, length); + varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); + varBinaryVector.setSafe(rowCount, data); } else { varBinaryVector.setNull(rowCount); } + } + } From 540f69bcab34d6e6cca2f36d8855c4bfd91dcc36 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 11 Apr 2018 09:31:25 -0400 Subject: [PATCH 076/129] File/s committed for @Parameterized Junit Test case related changes --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 2 +- .../org/apache/arrow/adapter/jdbc/Table.java | 85 ++++-- .../apache/arrow/adapter/jdbc/TestData.java | 37 +++ ...ues.java => JdbcToArrowTestIntValues.java} | 287 ++++++++---------- .../h2/JdbcToArrowTestMultipleDatatypes.java | 202 ++++++++++++ 5 files changed, 440 insertions(+), 173 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java rename java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/{TestIntValues.java => JdbcToArrowTestIntValues.java} (82%) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 818fa0365b6..98835615513 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -47,7 +47,7 @@ public static void assertIntVectorValues(IntVector intVector, int rowCount, int assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - assertEquals((int)values[j], intVector.get(j)); + assertEquals(values[j], intVector.get(j)); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 2888435d63d..648e075d445 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -18,66 +18,113 @@ package org.apache.arrow.adapter.jdbc; +import java.math.BigDecimal; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * POJO to handle the YAML data from the test YAML file. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class Table { private String name; private String create; - private String [] inserts; private String[] data; private String query; private String drop; - + + private int [] integers; + private int [] booleans; + private BigDecimal [] decimals; + private double [] doubles; + + private String intQuery; + private String booleanQuery; + private String decimalQuery; + private String doubleQuery; + public Table() { } - public String[] getInserts() { - return inserts; - } - - public void setInserts(String[] inserts) { - this.inserts = inserts; - } - public String getName() { return name; } - public void setName(String name) { this.name = name; } - public String getCreate() { return create; } - public void setCreate(String create) { this.create = create; } - public String[] getData() { return data; } - public void setData(String[] data) { this.data = data; } - public String getQuery() { return query; } - public void setQuery(String query) { this.query = query; } - public String getDrop() { return drop; } - public void setDrop(String drop) { this.drop = drop; } + + public int[] getIntegers() { + return integers; + } + public void setIntegers(int[] integers) { + this.integers = integers; + } + public int[] getBooleans() { + return booleans; + } + public void setBooleans(int[] booleans) { + this.booleans = booleans; + } + public BigDecimal [] getDecimals() { + return decimals; + } + public void setDecimals(BigDecimal [] decimals) { + this.decimals = decimals; + } + public double[] getDoubles() { + return doubles; + } + public void setDoubles(double[] doubles) { + this.doubles = doubles; + } + public String getIntQuery() { + return intQuery; + } + public void setIntQuery(String intQuery) { + this.intQuery = intQuery; + } + public String getBooleanQuery() { + return booleanQuery; + } + public void setBooleanQuery(String booleanQuery) { + this.booleanQuery = booleanQuery; + } + public String getDecimalQuery() { + return decimalQuery; + } + public void setDecimalQuery(String decimalQuery) { + this.decimalQuery = decimalQuery; + } + public String getDoubleQuery() { + return doubleQuery; + } + public void setDoubleQyery(String doubleQuery) { + this.doubleQuery = doubleQuery; + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java new file mode 100644 index 00000000000..b230d592a04 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java @@ -0,0 +1,37 @@ +package org.apache.arrow.adapter.jdbc; + +import java.math.BigDecimal; + +public class TestData { + + public int [] intValues; + public int [] boolValues; + public BigDecimal [] decimalValues; + public double [] doubleValues; + + public int[] getIntValues() { + return intValues; + } + public void setIntValues(int[] intValues) { + this.intValues = intValues; + } + public int[] getBoolValues() { + return boolValues; + } + public void setBoolValues(int[] boolValues) { + this.boolValues = boolValues; + } + public BigDecimal[] getDecimalValues() { + return decimalValues; + } + public void setDecimalValues(BigDecimal[] decimalValues) { + this.decimalValues = decimalValues; + } + public double[] getDoubleValues() { + return doubleValues; + } + public void setDoubleValues(double[] doubleValues) { + this.doubleValues = doubleValues; + } + +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java similarity index 82% rename from java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java rename to java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java index e07f6d83769..a1a05efc31d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/TestIntValues.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java @@ -1,153 +1,134 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.adapter.jdbc.h2; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.adapter.jdbc.Table; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Test; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; - -@RunWith(Parameterized.class) -public class TestIntValues { - private static Connection conn = null; - private static Table table; - - // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor - public int [] intValues; - - // This is the constructor which will add data to @Parameter annotated variable - public TestIntValues (IntegerValues integerValues) { - super(); - this.intValues = integerValues.intValues; - } - - // This is the method which is annotated for injecting values into the test class constructor - @Parameters - public static Collection getTestData() throws Exception { - // Calling setUp() method here in place of using it with @Before annotation because this method is called before @Before annotated - // method and we need Table object and also Connection object Because for this class creating table and populating it - // with data using YML file only which should be done automatically as per the comment so this will be change and it place of - // this we can call method to just create Table object (for ex - getTable(ymlFileNamePath)) - setUp(); - - final List valueList = new ArrayList<>(); - - Arrays.stream(table.getData()).parallel().forEach(value -> { - valueList.add(Integer.parseInt(value)); - }); - return Arrays.asList(new Object[][]{ - {new IntegerValues(valueList.stream().mapToInt(Integer::intValue).toArray())}}); - } - - // This is the test method after making changes as per comment - @Test - public void sqlToArrowTestInt() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // @Before - public static void setUp() throws Exception { - //try { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - - Class.forName(driver); - conn = DriverManager.getConnection(url); - - table = getTable ("h2/test1_int_h2.yml"); - createTestData(conn, table); - //} catch () - } - - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } - - protected static Table getTable (String ymlFilePath) { - Table table = null; - try { - table = new ObjectMapper(new YAMLFactory()).readValue( - TestIntValues.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } catch (JsonMappingException jme) { - jme.printStackTrace(); - } catch (JsonParseException jpe) { - jpe.printStackTrace(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - return table; - } - - protected static void createTestData(Connection conn, Table table) { - - Statement stmt = null; - try { - //create the table and insert the data and once done drop the table - stmt = conn.createStatement(); - stmt.executeUpdate(table.getCreate()); - - for (String insert: table.getInserts()) { - stmt.executeUpdate(insert); - } - - } catch (SQLException e) { - e.printStackTrace(); - } finally { - //JdbcToArrowUtils.closeStatement(stmt); - } - } - -} - -class IntegerValues { - - public int [] intValues; - - public IntegerValues(int [] intValues) { - this.intValues = intValues; - } -} +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.util.Arrays; +import java.util.Collection; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.adapter.jdbc.TestData; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; + +@RunWith(Parameterized.class) +public class JdbcToArrowTestIntValues { + private static Connection conn = null; + private static Table table; + + // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor + public int [] intValues; + + // This is the constructor which will add data to @Parameter annotated variable + public JdbcToArrowTestIntValues (TestData data) { + super(); + this.intValues = data.getIntValues(); + } + + // This is the method which is annotated for injecting values into the test class constructor + @Parameters + public static Collection getTestData() throws Exception { + // Calling setUp() method here in place of using it with @Before annotation because this method is called before @Before annotated + // method and we need Table object and also Connection object Because for this class creating table and populating it + // with data using YML file only which should be done automatically as per the comment so this will be change and it place of + // this we can call method to just create Table object (for ex - getTable(ymlFileNamePath)) + setUp(); + + TestData data = new TestData(); + data.setIntValues(table.getIntegers()); + + return Arrays.asList(new Object[][]{{data}}); + } + + // This is the test method after making changes as per comment + @Test + public void sqlToArrowTestInt() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + + Class.forName(driver); + conn = DriverManager.getConnection(url); + table = getTable ("h2/test1_int_h2.yml"); + } + + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + protected static Table getTable (String ymlFilePath) { + Table table = null; + try { + table = new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowTestIntValues.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } catch (JsonMappingException jme) { + jme.printStackTrace(); + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + return table; + } + + @Before + public void createTestData() { + Statement stmt = null; + try { + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + } catch (SQLException e) { + e.printStackTrace(); + } finally { + //JdbcToArrowUtils.closeStatement(stmt); + } + } +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java new file mode 100644 index 00000000000..092cc8164ae --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.util.Arrays; +import java.util.Collection; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.adapter.jdbc.TestData; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; + +@RunWith(Parameterized.class) +public class JdbcToArrowTestMultipleDatatypes { + private static Connection conn = null; + private static Table table; + // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor + public int [] intValues; + public int [] boolValues; + public BigDecimal [] decimalValues; + public double [] doubleValues; + + // This is the constructor which will add data to @Parameter annotated variable + public JdbcToArrowTestMultipleDatatypes (TestData data) { + super(); + this.intValues = data.getIntValues(); + this.boolValues = data.getBoolValues(); + this.decimalValues = data.getDecimalValues(); + this.doubleValues = data.getDoubleValues(); + } + + // This is the method which is annotated for injecting values into the test class constructor + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException { + setUp(); + + TestData data = new TestData(); + data.setIntValues(table.getIntegers()); + data.setBoolValues(table.getBooleans()); + data.setDecimalValues(table.getDecimals()); + data.setDoubleValues(table.getDoubles()); + + return Arrays.asList(new Object[][]{{data}}); + } + + // Calling individual test methods in this method, because need to call @After destroy() method only once to close connection + //finally after all the tests are performed, this is just temporarily done to show how can we perform multiple datatype tests + @Test + public void testDBValues() { + sqlToArrowTestInt(); + sqlToArrowTestBool(); + sqlToArrowTestBigDecimals(); + sqlToArrowTestDoubles(); + sqlToArrowTestMultipleDataTypes(); + } + + // Testing Int values + public void sqlToArrowTestInt() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getIntQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Testing Boolean values + public void sqlToArrowTestBool() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getBooleanQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Testing BigDecimal values + public void sqlToArrowTestBigDecimals() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getDecimalQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Testing Double values + public void sqlToArrowTestDoubles() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getDoubleQuery(), new RootAllocator(Integer.MAX_VALUE)); + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Testing Multiple Data Types values + public void sqlToArrowTestMultipleDataTypes() { + try { + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); + + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); + assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + + Class.forName(driver); + conn = DriverManager.getConnection(url); + table = getTable ("h2/test1_all_datatypes_h2.yml"); + } + + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + protected static Table getTable (String ymlFilePath) { + Table table = null; + try { + table = new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowTestMultipleDatatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } catch (JsonMappingException jme) { + jme.printStackTrace(); + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + return table; + } + + @Before + public void createTestData() { + + Statement stmt = null; + try { + //create the table and insert the data and once done drop the table + stmt = conn.createStatement(); + stmt.executeUpdate(table.getCreate()); + + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + + } catch (SQLException e) { + e.printStackTrace(); + } finally { + //JdbcToArrowUtils.closeStatement(stmt); + } + } +} + From 66f19e50890bf93228a4a2962b96c4431359a515 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 11 Apr 2018 09:32:49 -0400 Subject: [PATCH 077/129] File/s committed for @Parameterized Junit Test case related changes --- .../resources/h2/test1_all_datatypes_h2.yml | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index f610a80d376..3b1ac2a69ed 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -94,4 +94,78 @@ data: query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +integers: + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + +booleans: + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + +decimals: + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + +doubles: + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + +intQuery: 'select int_field1 from table1;' +booleanQuery: 'select bool_field2 from table1;' +decimalQuery: 'select decimal_field6 from table1;' +doubleQuery: 'select double_field7 from table1;' + \ No newline at end of file From 48308007a367ee3cfa08d47a230f669e088b3b82 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 11 Apr 2018 09:33:10 -0400 Subject: [PATCH 078/129] File/s committed for @Parameterized Junit Test case related changes --- .../jdbc/src/test/resources/h2/test1_int_h2.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml index d360773c55f..b8d06492ac1 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml @@ -62,6 +62,23 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' +integers: + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + - '101' + query: 'select int_field1 from table1;' drop: 'DROP table table1;' \ No newline at end of file From 675a570c5a9e2a99dd6b595aeb11118cf7617018 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 12 Apr 2018 13:38:06 -0700 Subject: [PATCH 079/129] Code changes based on PR review comments. --- java/adapter/jdbc/pom.xml | 15 +- .../arrow/adapter/jdbc/JdbcToArrow.java | 83 +++++++- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 199 ++++++++++++------ 3 files changed, 208 insertions(+), 89 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 7913220bf11..c67cab13589 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -38,7 +38,7 @@ com.google.guava guava - 18.0 + ${dep.guava.version} @@ -46,7 +46,7 @@ junit junit - 4.11 + ${dep.junit.version} test @@ -59,20 +59,13 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - 2.7.9 + ${dep.jackson.version} test com.fasterxml.jackson.core jackson-databind - 2.7.9 - test - - - - com.google.collections - google-collections - 1.0 + ${dep.jackson.version} test diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index e375a0264a8..db013e92682 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -18,6 +18,7 @@ package org.apache.arrow.adapter.jdbc; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; @@ -27,6 +28,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.ResultSet; +import java.util.Calendar; /** * Utility class to convert JDBC objects to columnar Arrow format objects. @@ -58,40 +60,98 @@ * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary * + * TODO: At this time, SQL Data type java.sql.Types.ARRAY is still not supported. + * * @since 0.10.0 */ public class JdbcToArrow { + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * This method uses the default Calendar instance with default TimeZone and Locale as returned by the JVM. + * If you wish to use specific TimeZone or Locale for any Date, Time and Timestamp datasets, you may want use + * overloaded API that taken Calendar object instance. + * + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. + */ + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + + return sqlToArrow(connection, query, allocator, Calendar.getInstance()); + } + /** * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. * * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), rootAllocator); + return sqlToArrow(stmt.executeQuery(query), allocator, calendar); } } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. This method + * uses the default RootAllocator and Calendar object. * * @param resultSet * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws Exception + * @throws SQLException */ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + return sqlToArrow(resultSet, Calendar.getInstance()); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param allocator Memory allocator + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + + return sqlToArrow(resultSet, allocator, Calendar.getInstance()); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator); + VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); rootAllocator.close(); return root; } @@ -100,16 +160,19 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. * * @param resultSet + * @param allocator Memory allocator to use. + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws Exception + * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(rootAllocator, "Root Allocator object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData()), rootAllocator); - JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root); + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), calendar), allocator); + JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, calendar); return root; } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 8b6a1c1be51..7fd1436f937 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -36,12 +36,25 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.holders.NullableBigIntHolder; +import org.apache.arrow.vector.holders.NullableBitHolder; +import org.apache.arrow.vector.holders.NullableDateMilliHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.holders.NullableFloat4Holder; +import org.apache.arrow.vector.holders.NullableFloat8Holder; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableSmallIntHolder; +import org.apache.arrow.vector.holders.NullableTimeMilliHolder; +import org.apache.arrow.vector.holders.NullableTinyIntHolder; +import org.apache.arrow.vector.holders.NullableVarBinaryHolder; +import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.DecimalUtility; import java.math.BigDecimal; @@ -56,6 +69,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; @@ -101,13 +115,14 @@ public class JdbcToArrowUtils { * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary * - * @param rsmd + * @param rsmd ResultSetMetaData * @return {@link Schema} * @throws SQLException */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); List fields = new ArrayList<>(); int columnCount = rsmd.getColumnCount(); @@ -158,8 +173,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); break; case Types.TIMESTAMP: - // TODO Need to handle timezone - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), null)); + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar.getTimeZone().getID())), null)); break; case Types.BINARY: case Types.VARBINARY: @@ -204,12 +218,13 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * * @param rs ResultSet to use to fetch the data from underlying database * @param root Arrow {@link VectorSchemaRoot} object to populate - * @throws Exception + * @throws SQLException */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); ResultSetMetaData rsmd = rs.getMetaData(); int columnCount = rsmd.getColumnCount(); @@ -218,46 +233,43 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int rowCount = 0; while (rs.next()) { - // for each column get the value based on the type - - // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { case Types.BOOLEAN: case Types.BIT: updateVector((BitVector)root.getVector(columnName), - rs.getBoolean(i), rowCount); + rs.getBoolean(i), !rs.wasNull(), rowCount); break; case Types.TINYINT: updateVector((TinyIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.SMALLINT: updateVector((SmallIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.INTEGER: updateVector((IntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.BIGINT: updateVector((BigIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getLong(i), !rs.wasNull(), rowCount); break; case Types.NUMERIC: case Types.DECIMAL: updateVector((DecimalVector)root.getVector(columnName), - rs.getBigDecimal(i), rowCount); + rs.getBigDecimal(i), !rs.wasNull(), rowCount); break; case Types.REAL: case Types.FLOAT: updateVector((Float4Vector)root.getVector(columnName), - rs.getFloat(i), rowCount); + rs.getFloat(i), !rs.wasNull(), rowCount); break; case Types.DOUBLE: updateVector((Float8Vector)root.getVector(columnName), - rs.getDouble(i), rowCount); + rs.getDouble(i), !rs.wasNull(), rowCount); break; case Types.CHAR: case Types.NCHAR: @@ -266,25 +278,26 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.LONGVARCHAR: case Types.LONGNVARCHAR: updateVector((VarCharVector)root.getVector(columnName), - rs.getString(i), rowCount); + rs.getString(i), !rs.wasNull(), rowCount); break; case Types.DATE: updateVector((DateMilliVector) root.getVector(columnName), - rs.getDate(i), rowCount); + rs.getDate(i, calendar), !rs.wasNull(), rowCount); break; case Types.TIME: updateVector((TimeMilliVector) root.getVector(columnName), - rs.getTime(i), rowCount); + rs.getTime(i, calendar), !rs.wasNull(), rowCount); break; case Types.TIMESTAMP: + // TODO: Need to handle precision such as milli, micro, nano updateVector((TimeStampVector)root.getVector(columnName), - rs.getTimestamp(i), rowCount); + rs.getTimestamp(i, calendar), !rs.wasNull(), rowCount); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBytes(i), rowCount); + rs.getBytes(i), !rs.wasNull(), rowCount); break; case Types.ARRAY: // TODO Need to handle this type @@ -292,11 +305,11 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.CLOB: updateVector((VarCharVector)root.getVector(columnName), - rs.getClob(i), rowCount); + rs.getClob(i), !rs.wasNull(), rowCount); break; case Types.BLOB: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBlob(i), rowCount); + rs.getBlob(i), !rs.wasNull(), rowCount); break; default: @@ -309,77 +322,127 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw root.setRowCount(rowCount); } - private static void updateVector(BitVector bitVector, boolean value, int rowCount) { - bitVector.setSafe(rowCount, value? 1: 0); + private static void updateVector(BitVector bitVector, boolean value, boolean isNonNull, int rowCount) { + NullableBitHolder holder = new NullableBitHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value ? 1 : 0; + } + bitVector.setSafe(rowCount, holder); bitVector.setValueCount(rowCount + 1); } - private static void updateVector(TinyIntVector tinyIntVector, int value, int rowCount) { - tinyIntVector.setSafe(rowCount, value); + private static void updateVector(TinyIntVector tinyIntVector, int value, boolean isNonNull, int rowCount) { + NullableTinyIntHolder holder = new NullableTinyIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = (byte) value; + } + tinyIntVector.setSafe(rowCount, holder); tinyIntVector.setValueCount(rowCount + 1); } - private static void updateVector(SmallIntVector smallIntVector, int value, int rowCount) { + private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { + NullableSmallIntHolder holder = new NullableSmallIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = (short) value; + } smallIntVector.setSafe(rowCount, value); smallIntVector.setValueCount(rowCount + 1); } - private static void updateVector(IntVector intVector, int value, int rowCount) { - intVector.setSafe(rowCount, value); + private static void updateVector(IntVector intVector, int value, boolean isNonNull, int rowCount) { + NullableIntHolder holder = new NullableIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + intVector.setSafe(rowCount, holder); intVector.setValueCount(rowCount + 1); } - private static void updateVector(BigIntVector bigIntVector, int value, int rowCount) { - bigIntVector.setSafe(rowCount, value); + private static void updateVector(BigIntVector bigIntVector, long value, boolean isNonNull, int rowCount) { + NullableBigIntHolder holder = new NullableBigIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + bigIntVector.setSafe(rowCount, holder); bigIntVector.setValueCount(rowCount + 1); } - private static void updateVector(DecimalVector decimalVector, BigDecimal value, int rowCount) { - decimalVector.setSafe(rowCount, value); + private static void updateVector(DecimalVector decimalVector, BigDecimal value, boolean isNonNull, int rowCount) { + NullableDecimalHolder holder = new NullableDecimalHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.precision = value.precision(); + holder.scale = value.scale(); + holder.buffer = decimalVector.getAllocator().buffer(DEFAULT_BUFFER_SIZE); + holder.start = 0; + DecimalUtility.writeBigDecimalToArrowBuf(value, holder.buffer, holder.start); + } + decimalVector.setSafe(rowCount, holder); decimalVector.setValueCount(rowCount + 1); } - private static void updateVector(Float4Vector float4Vector, float value, int rowCount) { - float4Vector.setSafe(rowCount, value); + private static void updateVector(Float4Vector float4Vector, float value, boolean isNonNull, int rowCount) { + NullableFloat4Holder holder = new NullableFloat4Holder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + float4Vector.setSafe(rowCount, holder); float4Vector.setValueCount(rowCount + 1); } - private static void updateVector(Float8Vector float8Vector, double value, int rowCount) { - float8Vector.setSafe(rowCount, value); + private static void updateVector(Float8Vector float8Vector, double value, boolean isNonNull, int rowCount) { + NullableFloat8Holder holder = new NullableFloat8Holder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + float8Vector.setSafe(rowCount, holder); float8Vector.setValueCount(rowCount + 1); } - private static void updateVector(VarCharVector varcharVector, String value, int rowCount) { - if (value != null) { - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, value.length()); - varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, value.length()); - varcharVector.setValueCount(rowCount + 1); + private static void updateVector(VarCharVector varcharVector, String value, boolean isNonNull, int rowCount) { + NullableVarCharHolder holder = new NullableVarCharHolder(); + holder.isSet = isNonNull? 1: 0; + varcharVector.setIndexDefined(rowCount); + if (isNonNull) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + holder.buffer = varcharVector.getAllocator().buffer(bytes.length); + holder.buffer.setBytes(0, bytes, 0, bytes.length); + holder.start = 0; + holder.end = bytes.length; } - // TODO: not sure how to handle null string value ??? + varcharVector.setSafe(rowCount, holder); + varcharVector.setValueCount(rowCount + 1); } - private static void updateVector(DateMilliVector dateMilliVector, Date date, int rowCount) { - //TODO: Need to handle Timezone - dateMilliVector.setValueCount(rowCount + 1); - if (date != null) { - dateMilliVector.setSafe(rowCount, date.getTime()); - } else { - dateMilliVector.setNull(rowCount); + private static void updateVector(DateMilliVector dateMilliVector, Date date, boolean isNonNull, int rowCount) { + NullableDateMilliHolder holder = new NullableDateMilliHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = date.getTime(); } + dateMilliVector.setSafe(rowCount, holder); + dateMilliVector.setValueCount(rowCount + 1); } - private static void updateVector(TimeMilliVector timeMilliVector, Time time, int rowCount) { - timeMilliVector.setValueCount(rowCount + 1); - if (time != null) { - timeMilliVector.setSafe(rowCount, (int) time.getTime()); - } else { - timeMilliVector.setNull(rowCount); + private static void updateVector(TimeMilliVector timeMilliVector, Time time, boolean isNonNull, int rowCount) { + NullableTimeMilliHolder holder = new NullableTimeMilliHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull && time != null) { + holder.value = (int)time.getTime(); } + timeMilliVector.setSafe(rowCount, holder); + timeMilliVector.setValueCount(rowCount + 1); } - private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, int rowCount) { - //TODO Need to handle timezone ??? + private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, boolean isNonNull, int rowCount) { + //TODO: Need to handle precision such as milli, micro, nano timeStampVector.setValueCount(rowCount + 1); if (timestamp != null) { timeStampVector.setSafe(rowCount, timestamp.getTime()); @@ -388,9 +451,9 @@ private static void updateVector(TimeStampVector timeStampVector, Timestamp time } } - private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, int rowCount) { + private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, boolean isNonNull, int rowCount) { varBinaryVector.setValueCount(rowCount + 1); - if (bytes != null) { + if (isNonNull && bytes != null) { varBinaryVector.setIndexDefined(rowCount); varBinaryVector.setValueLengthSafe(rowCount, bytes.length); varBinaryVector.setSafe(rowCount, bytes); @@ -399,9 +462,9 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, } } - private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { + private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException { varcharVector.setValueCount(rowCount + 1); - if (clob != null) { + if (isNonNull && clob != null) { int length = (int) clob.length(); String value = clob.getSubString(1, length); if (value != null) { @@ -416,9 +479,9 @@ private static void updateVector(VarCharVector varcharVector, Clob clob, int row } } - private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int rowCount) throws SQLException { + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException { varBinaryVector.setValueCount(rowCount + 1); - if (blob != null) { + if (isNonNull && blob != null) { byte[] data = blob.getBytes(0, (int) blob.length()); varBinaryVector.setIndexDefined(rowCount); varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); @@ -426,6 +489,6 @@ private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int } else { varBinaryVector.setNull(rowCount); } - } + } From 2b6b72092076c72d80d854f45e25dbccc4174012 Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 13 Apr 2018 13:44:52 -0400 Subject: [PATCH 080/129] Files committed for review comment implementation --- .../arrow/adapter/jdbc/JdbcToArrow.java | 17 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 199 ++++++++++++------ 2 files changed, 148 insertions(+), 68 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 7038e9a9b1e..650cc679c45 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -18,12 +18,12 @@ package org.apache.arrow.adapter.jdbc; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import com.google.common.base.Preconditions; - import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -69,15 +69,17 @@ public class JdbcToArrow { * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. * @param query The DB Query to fetch the data. + * @param allocator Memory allocator * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); - + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), rootAllocator); + return sqlToArrow(stmt.executeQuery(query), allocator); } } @@ -101,15 +103,16 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. * * @param resultSet + * @param allocator Memory allocator * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws Exception */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, RootAllocator rootAllocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(rootAllocator, "Root Allocator object can not be null"); + Preconditions.checkNotNull(allocator, "Root Allocator object can not be null"); VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData()), rootAllocator); + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData()), allocator); JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root); return root; } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 2745b7a728b..0a1867117cf 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -36,13 +36,29 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.impl.NullableVarCharHolderReaderImpl; +import org.apache.arrow.vector.holders.NullableBigIntHolder; +import org.apache.arrow.vector.holders.NullableBitHolder; +import org.apache.arrow.vector.holders.NullableDateMilliHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.holders.NullableFloat4Holder; +import org.apache.arrow.vector.holders.NullableFloat8Holder; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableSmallIntHolder; +import org.apache.arrow.vector.holders.NullableTimeMilliHolder; +import org.apache.arrow.vector.holders.NullableTinyIntHolder; +import org.apache.arrow.vector.holders.NullableVarBinaryHolder; +import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.DecimalUtility; + import java.math.BigDecimal; + import java.nio.charset.StandardCharsets; import java.sql.Blob; import java.sql.Clob; @@ -55,6 +71,7 @@ import java.sql.Types; import java.util.ArrayList; import java.util.List; + import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -98,7 +115,7 @@ public class JdbcToArrowUtils { * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary * - * @param rsmd + * @param rsmd ResultSetMetaData * @return {@link Schema} * @throws SQLException */ @@ -165,7 +182,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti break; case Types.ARRAY: // TODO Need to handle this type - // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); @@ -201,7 +218,7 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * * @param rs ResultSet to use to fetch the data from underlying database * @param root Arrow {@link VectorSchemaRoot} object to populate - * @throws Exception + * @throws SQLException */ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { @@ -216,6 +233,7 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int rowCount = 0; while (rs.next()) { // for each column get the value based on the type + // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); @@ -223,37 +241,37 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.BOOLEAN: case Types.BIT: updateVector((BitVector)root.getVector(columnName), - rs.getBoolean(i), rowCount); + rs.getBoolean(i), !rs.wasNull(), rowCount); break; case Types.TINYINT: updateVector((TinyIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.SMALLINT: updateVector((SmallIntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.INTEGER: updateVector((IntVector)root.getVector(columnName), - rs.getInt(i), rowCount); + rs.getInt(i), !rs.wasNull(), rowCount); break; case Types.BIGINT: updateVector((BigIntVector)root.getVector(columnName), - rs.getLong(i), rowCount); + rs.getLong(i), !rs.wasNull(), rowCount); break; case Types.NUMERIC: case Types.DECIMAL: updateVector((DecimalVector)root.getVector(columnName), - rs.getBigDecimal(i), rowCount); + rs.getBigDecimal(i), !rs.wasNull(), rowCount); break; case Types.REAL: case Types.FLOAT: updateVector((Float4Vector)root.getVector(columnName), - rs.getFloat(i), rowCount); + rs.getFloat(i), !rs.wasNull(), rowCount); break; case Types.DOUBLE: updateVector((Float8Vector)root.getVector(columnName), - rs.getDouble(i), rowCount); + rs.getDouble(i), !rs.wasNull(), rowCount); break; case Types.CHAR: case Types.NCHAR: @@ -262,19 +280,20 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw case Types.LONGVARCHAR: case Types.LONGNVARCHAR: updateVector((VarCharVector)root.getVector(columnName), - rs.getString(i), rowCount); + rs.getString(i), !rs.wasNull(), rowCount); break; case Types.DATE: updateVector((DateMilliVector) root.getVector(columnName), - rs.getDate(i), rowCount); + rs.getDate(i), !rs.wasNull(), rowCount); break; case Types.TIME: updateVector((TimeMilliVector) root.getVector(columnName), - rs.getTime(i), rowCount); + rs.getTime(i), !rs.wasNull(), rowCount); break; case Types.TIMESTAMP: + // TODO: Need to handle precision such as milli, micro, nano updateVector((TimeStampVector)root.getVector(columnName), - rs.getTimestamp(i), rowCount); + rs.getTimestamp(i), !rs.wasNull(), rowCount); break; case Types.BINARY: case Types.VARBINARY: @@ -284,15 +303,15 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.ARRAY: // TODO Need to handle this type - // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: - updateVector((VarCharVector)root.getVector(columnName), - rs.getClob(i), rowCount); + updateVector((VarCharVector)root.getVector(columnName), + rs.getClob(i), rowCount); break; case Types.BLOB: - updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBlob(i), rowCount); + updateVector((VarBinaryVector)root.getVector(columnName), + rs.getBlob(i), rowCount); break; default: @@ -305,77 +324,137 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw root.setRowCount(rowCount); } - private static void updateVector(BitVector bitVector, boolean value, int rowCount) { - bitVector.setSafe(rowCount, value? 1: 0); + private static void updateVector(BitVector bitVector, boolean value, boolean isNonNull, int rowCount) { + NullableBitHolder holder = new NullableBitHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value ? 1 : 0; + } + bitVector.setSafe(rowCount, holder); bitVector.setValueCount(rowCount + 1); } - private static void updateVector(TinyIntVector tinyIntVector, int value, int rowCount) { - tinyIntVector.setSafe(rowCount, value); + private static void updateVector(TinyIntVector tinyIntVector, int value, boolean isNonNull, int rowCount) { + NullableTinyIntHolder holder = new NullableTinyIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = (byte) value; + } + tinyIntVector.setSafe(rowCount, holder); tinyIntVector.setValueCount(rowCount + 1); } - private static void updateVector(SmallIntVector smallIntVector, int value, int rowCount) { - smallIntVector.setSafe(rowCount, value); + private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { + NullableSmallIntHolder holder = new NullableSmallIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = (short) value; + } + smallIntVector.setSafe(rowCount, holder); smallIntVector.setValueCount(rowCount + 1); } - private static void updateVector(IntVector intVector, int value, int rowCount) { - intVector.setSafe(rowCount, value); + private static void updateVector(IntVector intVector, int value, boolean isNonNull, int rowCount) { + NullableIntHolder holder = new NullableIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + intVector.setSafe(rowCount, holder); intVector.setValueCount(rowCount + 1); } - private static void updateVector(BigIntVector bigIntVector, long value, int rowCount) { - bigIntVector.setSafe(rowCount, value); + private static void updateVector(BigIntVector bigIntVector, long value, boolean isNonNull, int rowCount) { + NullableBigIntHolder holder = new NullableBigIntHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + bigIntVector.setSafe(rowCount, holder); bigIntVector.setValueCount(rowCount + 1); } - private static void updateVector(DecimalVector decimalVector, BigDecimal value, int rowCount) { - decimalVector.setSafe(rowCount, value); + private static void updateVector(DecimalVector decimalVector, BigDecimal value, boolean isNonNull, int rowCount) { + NullableDecimalHolder holder = new NullableDecimalHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.precision = value.precision(); + holder.scale = value.scale(); + holder.buffer = decimalVector.getAllocator().buffer(DEFAULT_BUFFER_SIZE); + holder.start = 0; + DecimalUtility.writeBigDecimalToArrowBuf(value, holder.buffer, holder.start); + } + decimalVector.setSafe(rowCount, holder); decimalVector.setValueCount(rowCount + 1); } - private static void updateVector(Float4Vector float4Vector, float value, int rowCount) { - float4Vector.setSafe(rowCount, value); + private static void updateVector(Float4Vector float4Vector, float value, boolean isNonNull, int rowCount) { + NullableFloat4Holder holder = new NullableFloat4Holder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + float4Vector.setSafe(rowCount, holder); float4Vector.setValueCount(rowCount + 1); } - private static void updateVector(Float8Vector float8Vector, double value, int rowCount) { - float8Vector.setSafe(rowCount, value); + private static void updateVector(Float8Vector float8Vector, double value, boolean isNonNull, int rowCount) { + NullableFloat8Holder holder = new NullableFloat8Holder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = value; + } + float8Vector.setSafe(rowCount, holder); float8Vector.setValueCount(rowCount + 1); } - private static void updateVector(VarCharVector varcharVector, String value, int rowCount) { - if (value != null) { - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, value.length()); - varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, value.length()); - varcharVector.setValueCount(rowCount + 1); + private static void updateVector(VarCharVector varcharVector, String value, boolean isNonNull, int rowCount) { + NullableVarCharHolder holder = new NullableVarCharHolder(); + holder.isSet = isNonNull? 1: 0; + varcharVector.setIndexDefined(rowCount); + + if (isNonNull) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + int bytesLength = bytes.length; + + holder.buffer = varcharVector.getAllocator().buffer(bytesLength); + holder.start = 0; + holder.end = bytesLength; + holder.buffer.setBytes(0, bytes, 0, bytesLength); + } else { + holder.buffer = varcharVector.getAllocator().buffer(0); } - // TODO: not sure how to handle null string value ??? + + varcharVector.setSafe(rowCount, holder); + varcharVector.setValueCount(rowCount + 1); + } - private static void updateVector(DateMilliVector dateMilliVector, Date date, int rowCount) { + private static void updateVector(DateMilliVector dateMilliVector, Date date, boolean isNonNull, int rowCount) { //TODO: Need to handle Timezone - dateMilliVector.setValueCount(rowCount + 1); - if (date != null) { - dateMilliVector.setSafe(rowCount, date.getTime()); - } else { - dateMilliVector.setNull(rowCount); + NullableDateMilliHolder holder = new NullableDateMilliHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = date.getTime(); } + dateMilliVector.setSafe(rowCount, holder); + dateMilliVector.setValueCount(rowCount + 1); } - private static void updateVector(TimeMilliVector timeMilliVector, Time time, int rowCount) { - timeMilliVector.setValueCount(rowCount + 1); - if (time != null) { - timeMilliVector.setSafe(rowCount, (int) time.getTime()); - } else { - timeMilliVector.setNull(rowCount); + private static void updateVector(TimeMilliVector timeMilliVector, Time time, boolean isNonNull, int rowCount) { + //TODO: Need to handle Timezone + NullableTimeMilliHolder holder = new NullableTimeMilliHolder(); + holder.isSet = isNonNull? 1: 0; + if (isNonNull) { + holder.value = (int)time.getTime(); } + timeMilliVector.setSafe(rowCount, holder); + timeMilliVector.setValueCount(rowCount + 1); } - private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, int rowCount) { - //TODO Need to handle timezone ??? + private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, boolean isNonNull, int rowCount) { + //TODO: Need to handle timezone + //TODO: Need to handle precision such as milli, micro, nano timeStampVector.setValueCount(rowCount + 1); if (timestamp != null) { timeStampVector.setSafe(rowCount, timestamp.getTime()); @@ -394,7 +473,7 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, varBinaryVector.setNull(rowCount); } } - + private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { varcharVector.setValueCount(rowCount + 1); if (clob != null) { @@ -422,8 +501,6 @@ private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int } else { varBinaryVector.setNull(rowCount); } - } - - + } From 32bfb1f3767cf5e66a0d3032bc6b54cd33a3789c Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 13 Apr 2018 13:45:48 -0400 Subject: [PATCH 081/129] Files committed for @Parameterized testcase changes --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 98835615513..e3deb98f78d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -21,6 +21,8 @@ import java.math.BigDecimal; import java.util.Arrays; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateMilliVector; @@ -34,8 +36,8 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; - import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; /** * This is a Helper class which has functionalities to read and assert the values from the given FieldVector object @@ -47,10 +49,10 @@ public static void assertIntVectorValues(IntVector intVector, int rowCount, int assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - assertEquals(values[j], intVector.get(j)); - } + assertEquals(values[j], intVector.get(j)); + } } - + public static void assertBitBooleanVectorValues(BitVector bitVector, int rowCount, int[] values){ assertEquals(rowCount, bitVector.getValueCount()); @@ -153,9 +155,17 @@ public static void assertVarcharVectorValues(VarCharVector varCharVector, int ro } } catch (AssertionError ae) { ae.printStackTrace(); - } + } } + + public static void assertNullValues(BaseValueVector vector, int rowCount) { + assertEquals(rowCount, vector.getValueCount()); + for(int j = 0; j < vector.getValueCount(); j++) { + assertTrue(vector.isNull(j)); + } + } + public static byte[] hexStringToByteArray(String s) { int len = s.length(); byte[] data = new byte[len / 2]; From bafbd1af4c87eeeadc1f22b16269ac959549bbdb Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 13 Apr 2018 13:46:11 -0400 Subject: [PATCH 082/129] Files committed for @Parameterized testcase changes --- .../org/apache/arrow/adapter/jdbc/Table.java | 165 ++++++++++++++---- 1 file changed, 135 insertions(+), 30 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 648e075d445..ba7cae2632b 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -19,7 +19,12 @@ package org.apache.arrow.adapter.jdbc; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -34,17 +39,23 @@ public class Table { private String[] data; private String query; private String drop; - - private int [] integers; + private int [] ints; private int [] booleans; - private BigDecimal [] decimals; + private int [] tinyInts; + private int [] smallInts; + private int [] bigInts; + private long [] times; + private long [] dates; + private long [] timestamps; + private String [] bytes; + private String [] varchars; + private String [] chars; + private String [] clobs; + private float [] reals; private double [] doubles; - - private String intQuery; - private String booleanQuery; - private String decimalQuery; - private String doubleQuery; - + private BigDecimal [] decimals; + private String [] selectQuereis; + public Table() { } @@ -78,12 +89,11 @@ public String getDrop() { public void setDrop(String drop) { this.drop = drop; } - - public int[] getIntegers() { - return integers; + public int[] getInts() { + return ints; } - public void setIntegers(int[] integers) { - this.integers = integers; + public void setInts(int[] ints) { + this.ints = ints; } public int[] getBooleans() { return booleans; @@ -103,28 +113,123 @@ public double[] getDoubles() { public void setDoubles(double[] doubles) { this.doubles = doubles; } - public String getIntQuery() { - return intQuery; + public int[] getTinyInts() { + return tinyInts; + } + public void setTinyInts(int[] tinyInts) { + this.tinyInts = tinyInts; + } + public int[] getSmallInts() { + return smallInts; + } + public void setSmallInts(int[] smallInts) { + this.smallInts = smallInts; + } + public int[] getBigInts() { + return bigInts; + } + public void setBigInts(int[] bigInts) { + this.bigInts = bigInts; + } + public long[] getTimes() { + return times; + } + public void setTimes(long[] times) { + this.times = times; + } + public long[] getDates() { + return dates; + } + public void setDates(long[] dates) { + this.dates = dates; + } + public long[] getTimestamps() { + return timestamps; + } + public void setTimestamps(long[] timestamps) { + this.timestamps = timestamps; + } + public String [] getBytes() { + return bytes; + } + public void setBytes(String [] bytes) { + this.bytes = bytes; } - public void setIntQuery(String intQuery) { - this.intQuery = intQuery; + public String []getVarchars() { + return varchars; } - public String getBooleanQuery() { - return booleanQuery; + public void setVarchars(String [] varchars) { + this.varchars = varchars; } - public void setBooleanQuery(String booleanQuery) { - this.booleanQuery = booleanQuery; + public String [] getChars() { + return chars; } - public String getDecimalQuery() { - return decimalQuery; + public void setChars(String [] chars) { + this.chars = chars; } - public void setDecimalQuery(String decimalQuery) { - this.decimalQuery = decimalQuery; + public String [] getClobs() { + return clobs; } - public String getDoubleQuery() { - return doubleQuery; + public void setClobs(String [] clobs) { + this.clobs = clobs; } - public void setDoubleQyery(String doubleQuery) { - this.doubleQuery = doubleQuery; + public float[] getReals() { + return reals; } + public void setReals(float[] reals) { + this.reals = reals; + } + public String[] getSelectQuereis() { + return selectQuereis; + } + public void setSelectQuereis(String[] selectQuereis) { + this.selectQuereis = selectQuereis; + } + + public String getSelectQuery(String columnName) { + String queryString = "select " + columnName + " from"; + String query = Arrays.stream(selectQuereis).parallel().filter(q -> q.toUpperCase().contains(queryString.toUpperCase())).findFirst().get(); + return query; + } + + public byte [][] getHexStringAsByte () { + return getHexToByteArray (bytes); + } + public byte [][] getClobAsByte () { + return getByteArray (clobs); + } + public byte [][] getCharAsByte () { + return getByteArray (chars); + } + public byte [][] getVarCharAsByte () { + return getByteArray (varchars); + } + + private byte [][] getByteArray (String [] data) { + byte [][] byteArr = new byte [data.length][]; + + for (int i = 0; i < data.length; i++) { + byteArr [i] = data [i].getBytes(StandardCharsets.UTF_8); + } + return byteArr; + } + + private byte [][] getHexToByteArray (String [] data){ + byte [][] byteArr = new byte [data.length][]; + + for (int i = 0; i < data.length; i++) { + byteArr [i] = hexStringToByteArray(data [i]); + } + return byteArr; + } + + private static byte[] hexStringToByteArray(String s) { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + + Character.digit(s.charAt(i+1), 16)); + } + return data; + } } From 09b526e7e103a551035f72e12a1bd935a2788faf Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 13 Apr 2018 13:50:22 -0400 Subject: [PATCH 083/129] Files committed for @Parameterized testcase changes --- .../jdbc/h2/JdbcToArrowTestDatatypes.java | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java new file mode 100644 index 00000000000..92f8b858c68 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.util.Arrays; +import java.util.Collection; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; + +@RunWith(Parameterized.class) +public class JdbcToArrowTestDatatypes { + private static Connection conn = null; + private static Table table; + private int [] intValues; + private int [] boolValues; + private BigDecimal [] decimalValues; + private double [] doubleValues; + private int [] tinyIntValues; + private int [] smallIntValues; + private int [] bigIntValues; + private long [] timeValues; + private long [] dateValues; + private long [] timestampValues; + private float [] realValues; + private byte [][] byteValues; + private byte [][] varCharValues; + private byte [][] charValues; + private byte [][] clobValues; + + public JdbcToArrowTestDatatypes (Table data) { + super(); + this.intValues = data.getInts(); + this.boolValues = data.getBooleans(); + this.decimalValues = data.getDecimals(); + this.doubleValues = data.getDoubles(); + this.tinyIntValues = data.getTinyInts(); + this.smallIntValues = data.getSmallInts(); + this.bigIntValues = data.getBigInts(); + this.timeValues = data.getTimes(); + this.dateValues = data.getDates(); + this.timestampValues = data.getTimestamps(); + this.realValues = data.getReals(); + this.byteValues = data.getHexStringAsByte(); + this.varCharValues = data.getVarCharAsByte(); + this.charValues = data.getCharAsByte(); + this.clobValues = data.getClobAsByte(); + + } + + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException { + setUp(); + return Arrays.asList(new Object[][]{{table}}); + } + + @Test + public void testDBValues() { + try { + sqlToArrowTestInt(); + sqlToArrowTestBool(); + sqlToArrowTestTinyInts(); + sqlToArrowTestSmallInts(); + sqlToArrowTestBigInts(); + sqlToArrowTestBigDecimals(); + sqlToArrowTestDoubles(); + sqlToArrowTestRealValues(); + sqlToArrowTestTimeValues(); + sqlToArrowTestDateValues(); + sqlToArrowTestTimestampValues(); + sqlToArrowTestByteValues(); + sqlToArrowTestVarCharValues(); + sqlToArrowTestCharValues(); + sqlToArrowTestBlobValues(); + sqlToArrowTestClobValues(); + sqlToArrowTestBits(); + sqlToArrowTestNullValues(); + } catch (SQLException sqe) { + sqe.printStackTrace(); + } + } + + public void sqlToArrowTestInt() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1", true), new RootAllocator(Integer.MAX_VALUE))){ + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + } + } + + public void sqlToArrowTestBool() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); + } + } + + public void sqlToArrowTestTinyInts() throws SQLException { + try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); + } + } + + public void sqlToArrowTestSmallInts() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4", true), new RootAllocator(Integer.MAX_VALUE))){ + assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); + } + } + + public void sqlToArrowTestBigInts() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIGINT_FIELD5", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); + } + } + + public void sqlToArrowTestBigDecimals() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DECIMAL_FIELD6", true), new RootAllocator(Integer.MAX_VALUE))){ + assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); + } + } + + public void sqlToArrowTestDoubles() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DOUBLE_FIELD7", true), new RootAllocator(Integer.MAX_VALUE))){ + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); + } + } + + public void sqlToArrowTestRealValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("REAL_FIELD8", true), new RootAllocator(Integer.MAX_VALUE))){ + assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); + } + } + + public void sqlToArrowTestTimeValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIME_FIELD9", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); + } + } + + public void sqlToArrowTestDateValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DATE_FIELD10", true), new RootAllocator(Integer.MAX_VALUE))){ + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); + } + } + + public void sqlToArrowTestTimestampValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIMESTAMP_FIELD11", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); + } + } + + public void sqlToArrowTestByteValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BINARY_FIELD12", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); + + } + } + + public void sqlToArrowTestVarCharValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("VARCHAR_FIELD13", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); + } + } + + public void sqlToArrowTestBlobValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BLOB_FIELD14", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); + } + } + + public void sqlToArrowTestClobValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CLOB_FIELD15", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); + } + } + + public void sqlToArrowTestCharValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CHAR_FIELD16", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); + } + } + + public void sqlToArrowTestBits() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIT_FIELD17", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); + } + } + + public void sqlToArrowTestNullValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("all", false, "INT_FIELD1"), new RootAllocator(Integer.MAX_VALUE))){ + assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); + assertNullValues((BitVector)root.getVector("BOOL_FIELD2"), 5); + assertNullValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 5); + assertNullValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 5); + assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); + assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); + assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); + assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); + assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); + assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); + assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); + assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); + assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); + assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); + assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); + assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); + assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); + } + } + + @Before + public void createTestData() throws SQLException{ + try (Statement stmt = conn.createStatement();){ + stmt.executeUpdate(table.getCreate()); + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + private static void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + table = getTable ("h2/test1_all_datatypes_h2.yml"); + } + + private static Table getTable (String ymlFilePath) { + Table table = null; + try { + table = new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowTestDatatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } catch (JsonMappingException jme) { + jme.printStackTrace(); + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + return table; + } + + private String getQuery (String columnName, boolean isNotNull) { + return getQuery (null, isNotNull, columnName); + } + + private String getQuery (String allColumns, boolean isNotNull, String column) { + StringBuffer query = new StringBuffer(allColumns != null ? table.getSelectQuery("*") : table.getSelectQuery(column)); + query.append(isNotNull ? " where " + column + " is not null;" : " where " + column + " is null;"); + return query.toString(); + } +} + From 7320cdc7186702c9ba8840e4a2813d3979c9fced Mon Sep 17 00:00:00 2001 From: yashpal Date: Fri, 13 Apr 2018 13:50:58 -0400 Subject: [PATCH 084/129] Files committed for @Parameterized testcase changes --- .../resources/h2/test1_all_datatypes_h2.yml | 220 +++++++++++++++++- 1 file changed, 213 insertions(+), 7 deletions(-) diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 3b1ac2a69ed..0a8ab5ae89f 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -90,13 +90,18 @@ data: PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, - time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1;' + time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' drop: 'DROP table table1;' -integers: +ints: - '101' - '101' - '101' @@ -130,6 +135,74 @@ booleans: - '1' - '1' +tinyInts: + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + +smallInts: + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + +bigInts: + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + +reals: + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + decimals: - '17345667789.23' - '17345667789.23' @@ -164,8 +237,141 @@ doubles: - '56478356785.345' - '56478356785.345' -intQuery: 'select int_field1 from table1;' -booleanQuery: 'select bool_field2 from table1;' -decimalQuery: 'select decimal_field6 from table1;' -doubleQuery: 'select double_field7 from table1;' - \ No newline at end of file +times: + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + +dates: + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + +timestamps: + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + +bytes: + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + +varchars: + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + +clobs: + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + +chars: + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + +selectQuereis: + - 'select int_field1 from table1' + - 'select bool_field2 from table1' + - 'select tinyint_field3 from table1' + - 'select smallint_field4 from table1' + - 'select bigint_field5 from table1' + - 'select decimal_field6 from table1' + - 'select double_field7 from table1' + - 'select real_field8 from table1' + - 'select time_field9 from table1' + - 'select date_field10 from table1' + - 'select timestamp_field11 from table1' + - 'select binary_field12 from table1' + - 'select varchar_field13 from table1' + - 'select blob_field14 from table1' + - 'select clob_field15 from table1' + - 'select char_field16 from table1' + - 'select bit_field17 from table1' + - 'select * from table1' \ No newline at end of file From 0707a0c6d7adaa6550ff15c1de3af1e5f62195c3 Mon Sep 17 00:00:00 2001 From: YashpalThakur <36757209+YashpalThakur@users.noreply.github.com> Date: Mon, 16 Apr 2018 13:08:25 +0530 Subject: [PATCH 085/129] Delete TestData.java file as it is not required --- .../apache/arrow/adapter/jdbc/TestData.java | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java deleted file mode 100644 index b230d592a04..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/TestData.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.apache.arrow.adapter.jdbc; - -import java.math.BigDecimal; - -public class TestData { - - public int [] intValues; - public int [] boolValues; - public BigDecimal [] decimalValues; - public double [] doubleValues; - - public int[] getIntValues() { - return intValues; - } - public void setIntValues(int[] intValues) { - this.intValues = intValues; - } - public int[] getBoolValues() { - return boolValues; - } - public void setBoolValues(int[] boolValues) { - this.boolValues = boolValues; - } - public BigDecimal[] getDecimalValues() { - return decimalValues; - } - public void setDecimalValues(BigDecimal[] decimalValues) { - this.decimalValues = decimalValues; - } - public double[] getDoubleValues() { - return doubleValues; - } - public void setDoubleValues(double[] doubleValues) { - this.doubleValues = doubleValues; - } - -} From af439433e78acc1b2cc7ffb929da84403b3013fe Mon Sep 17 00:00:00 2001 From: YashpalThakur <36757209+YashpalThakur@users.noreply.github.com> Date: Mon, 16 Apr 2018 13:09:20 +0530 Subject: [PATCH 086/129] Removed this file as new one is created --- .../h2/JdbcToArrowTestMultipleDatatypes.java | 202 ------------------ 1 file changed, 202 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java deleted file mode 100644 index 092cc8164ae..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestMultipleDatatypes.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.adapter.jdbc.h2; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.util.Arrays; -import java.util.Collection; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.adapter.jdbc.Table; -import org.apache.arrow.adapter.jdbc.TestData; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; - -@RunWith(Parameterized.class) -public class JdbcToArrowTestMultipleDatatypes { - private static Connection conn = null; - private static Table table; - // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor - public int [] intValues; - public int [] boolValues; - public BigDecimal [] decimalValues; - public double [] doubleValues; - - // This is the constructor which will add data to @Parameter annotated variable - public JdbcToArrowTestMultipleDatatypes (TestData data) { - super(); - this.intValues = data.getIntValues(); - this.boolValues = data.getBoolValues(); - this.decimalValues = data.getDecimalValues(); - this.doubleValues = data.getDoubleValues(); - } - - // This is the method which is annotated for injecting values into the test class constructor - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException { - setUp(); - - TestData data = new TestData(); - data.setIntValues(table.getIntegers()); - data.setBoolValues(table.getBooleans()); - data.setDecimalValues(table.getDecimals()); - data.setDoubleValues(table.getDoubles()); - - return Arrays.asList(new Object[][]{{data}}); - } - - // Calling individual test methods in this method, because need to call @After destroy() method only once to close connection - //finally after all the tests are performed, this is just temporarily done to show how can we perform multiple datatype tests - @Test - public void testDBValues() { - sqlToArrowTestInt(); - sqlToArrowTestBool(); - sqlToArrowTestBigDecimals(); - sqlToArrowTestDoubles(); - sqlToArrowTestMultipleDataTypes(); - } - - // Testing Int values - public void sqlToArrowTestInt() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getIntQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // Testing Boolean values - public void sqlToArrowTestBool() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getBooleanQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // Testing BigDecimal values - public void sqlToArrowTestBigDecimals() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getDecimalQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // Testing Double values - public void sqlToArrowTestDoubles() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getDoubleQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // Testing Multiple Data Types values - public void sqlToArrowTestMultipleDataTypes() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); - - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); - assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - - Class.forName(driver); - conn = DriverManager.getConnection(url); - table = getTable ("h2/test1_all_datatypes_h2.yml"); - } - - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } - - protected static Table getTable (String ymlFilePath) { - Table table = null; - try { - table = new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowTestMultipleDatatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } catch (JsonMappingException jme) { - jme.printStackTrace(); - } catch (JsonParseException jpe) { - jpe.printStackTrace(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - return table; - } - - @Before - public void createTestData() { - - Statement stmt = null; - try { - //create the table and insert the data and once done drop the table - stmt = conn.createStatement(); - stmt.executeUpdate(table.getCreate()); - - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - - } catch (SQLException e) { - e.printStackTrace(); - } finally { - //JdbcToArrowUtils.closeStatement(stmt); - } - } -} - From f3b923c9a7016ac78a1322fc15878cfab1721cb4 Mon Sep 17 00:00:00 2001 From: YashpalThakur <36757209+YashpalThakur@users.noreply.github.com> Date: Mon, 16 Apr 2018 13:10:04 +0530 Subject: [PATCH 087/129] removed this file as this is no longer required. --- .../jdbc/h2/JdbcToArrowTestIntValues.java | 134 ------------------ 1 file changed, 134 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java deleted file mode 100644 index a1a05efc31d..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestIntValues.java +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.adapter.jdbc.h2; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.util.Arrays; -import java.util.Collection; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.adapter.jdbc.Table; -import org.apache.arrow.adapter.jdbc.TestData; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; - -@RunWith(Parameterized.class) -public class JdbcToArrowTestIntValues { - private static Connection conn = null; - private static Table table; - - // This is the parameter which will be populated with data from YAML file using @Parameters annotated method via constructor - public int [] intValues; - - // This is the constructor which will add data to @Parameter annotated variable - public JdbcToArrowTestIntValues (TestData data) { - super(); - this.intValues = data.getIntValues(); - } - - // This is the method which is annotated for injecting values into the test class constructor - @Parameters - public static Collection getTestData() throws Exception { - // Calling setUp() method here in place of using it with @Before annotation because this method is called before @Before annotated - // method and we need Table object and also Connection object Because for this class creating table and populating it - // with data using YML file only which should be done automatically as per the comment so this will be change and it place of - // this we can call method to just create Table object (for ex - getTable(ymlFileNamePath)) - setUp(); - - TestData data = new TestData(); - data.setIntValues(table.getIntegers()); - - return Arrays.asList(new Object[][]{{data}}); - } - - // This is the test method after making changes as per comment - @Test - public void sqlToArrowTestInt() { - try { - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)); - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - - Class.forName(driver); - conn = DriverManager.getConnection(url); - table = getTable ("h2/test1_int_h2.yml"); - } - - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } - - protected static Table getTable (String ymlFilePath) { - Table table = null; - try { - table = new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowTestIntValues.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } catch (JsonMappingException jme) { - jme.printStackTrace(); - } catch (JsonParseException jpe) { - jpe.printStackTrace(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - return table; - } - - @Before - public void createTestData() { - Statement stmt = null; - try { - //create the table and insert the data and once done drop the table - stmt = conn.createStatement(); - stmt.executeUpdate(table.getCreate()); - - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - } catch (SQLException e) { - e.printStackTrace(); - } finally { - //JdbcToArrowUtils.closeStatement(stmt); - } - } -} From 6578d17c7654e3f6c2d185be9840ad9aa258c2fe Mon Sep 17 00:00:00 2001 From: yashpal Date: Mon, 16 Apr 2018 03:43:58 -0400 Subject: [PATCH 088/129] file committed after renaming it --- .../jdbc/h2/JdbcToArrowTestH2Datatypes.java | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java new file mode 100644 index 00000000000..55fdbe5e71c --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.util.Arrays; +import java.util.Collection; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; + +@RunWith(Parameterized.class) +public class JdbcToArrowTestH2Datatypes { + private static Connection conn = null; + private static Table table; + private int [] intValues; + private int [] boolValues; + private BigDecimal [] decimalValues; + private double [] doubleValues; + private int [] tinyIntValues; + private int [] smallIntValues; + private int [] bigIntValues; + private long [] timeValues; + private long [] dateValues; + private long [] timestampValues; + private float [] realValues; + private byte [][] byteValues; + private byte [][] varCharValues; + private byte [][] charValues; + private byte [][] clobValues; + + public JdbcToArrowTestH2Datatypes (Table data) { + super(); + this.intValues = data.getInts(); + this.boolValues = data.getBooleans(); + this.decimalValues = data.getDecimals(); + this.doubleValues = data.getDoubles(); + this.tinyIntValues = data.getTinyInts(); + this.smallIntValues = data.getSmallInts(); + this.bigIntValues = data.getBigInts(); + this.timeValues = data.getTimes(); + this.dateValues = data.getDates(); + this.timestampValues = data.getTimestamps(); + this.realValues = data.getReals(); + this.byteValues = data.getHexStringAsByte(); + this.varCharValues = data.getVarCharAsByte(); + this.charValues = data.getCharAsByte(); + this.clobValues = data.getClobAsByte(); + + } + + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException { + setUp(); + return Arrays.asList(new Object[][]{{table}}); + } + + @Test + public void testDBValues() { + try { + sqlToArrowTestInt(); + sqlToArrowTestBool(); + sqlToArrowTestTinyInts(); + sqlToArrowTestSmallInts(); + sqlToArrowTestBigInts(); + sqlToArrowTestBigDecimals(); + sqlToArrowTestDoubles(); + sqlToArrowTestRealValues(); + sqlToArrowTestTimeValues(); + sqlToArrowTestDateValues(); + sqlToArrowTestTimestampValues(); + sqlToArrowTestByteValues(); + sqlToArrowTestVarCharValues(); + sqlToArrowTestCharValues(); + sqlToArrowTestBlobValues(); + sqlToArrowTestClobValues(); + sqlToArrowTestBits(); + sqlToArrowTestNullValues(); + } catch (SQLException sqe) { + sqe.printStackTrace(); + } + } + + public void sqlToArrowTestInt() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1", true), new RootAllocator(Integer.MAX_VALUE))){ + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); + } + } + + public void sqlToArrowTestBool() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); + } + } + + public void sqlToArrowTestTinyInts() throws SQLException { + try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); + } + } + + public void sqlToArrowTestSmallInts() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4", true), new RootAllocator(Integer.MAX_VALUE))){ + assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); + } + } + + public void sqlToArrowTestBigInts() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIGINT_FIELD5", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); + } + } + + public void sqlToArrowTestBigDecimals() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DECIMAL_FIELD6", true), new RootAllocator(Integer.MAX_VALUE))){ + assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); + } + } + + public void sqlToArrowTestDoubles() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DOUBLE_FIELD7", true), new RootAllocator(Integer.MAX_VALUE))){ + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); + } + } + + public void sqlToArrowTestRealValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("REAL_FIELD8", true), new RootAllocator(Integer.MAX_VALUE))){ + assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); + } + } + + public void sqlToArrowTestTimeValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIME_FIELD9", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); + } + } + + public void sqlToArrowTestDateValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DATE_FIELD10", true), new RootAllocator(Integer.MAX_VALUE))){ + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); + } + } + + public void sqlToArrowTestTimestampValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIMESTAMP_FIELD11", true), new RootAllocator(Integer.MAX_VALUE))){ + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); + } + } + + public void sqlToArrowTestByteValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BINARY_FIELD12", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); + + } + } + + public void sqlToArrowTestVarCharValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("VARCHAR_FIELD13", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); + } + } + + public void sqlToArrowTestBlobValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BLOB_FIELD14", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); + } + } + + public void sqlToArrowTestClobValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CLOB_FIELD15", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); + } + } + + public void sqlToArrowTestCharValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CHAR_FIELD16", true), new RootAllocator(Integer.MAX_VALUE))){ + assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); + } + } + + public void sqlToArrowTestBits() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIT_FIELD17", true), new RootAllocator(Integer.MAX_VALUE))){ + assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); + } + } + + public void sqlToArrowTestNullValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("all", false, "INT_FIELD1"), new RootAllocator(Integer.MAX_VALUE))){ + assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); + assertNullValues((BitVector)root.getVector("BOOL_FIELD2"), 5); + assertNullValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 5); + assertNullValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 5); + assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); + assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); + assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); + assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); + assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); + assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); + assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); + assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); + assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); + assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); + assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); + assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); + assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); + } + } + + @Before + public void createTestData() throws SQLException{ + try (Statement stmt = conn.createStatement();){ + stmt.executeUpdate(table.getCreate()); + for (String insert: table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + private static void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + table = getTable ("h2/test1_all_datatypes_h2.yml"); + } + + private static Table getTable (String ymlFilePath) { + Table table = null; + try { + table = new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowTestH2Datatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } catch (JsonMappingException jme) { + jme.printStackTrace(); + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + return table; + } + + private String getQuery (String columnName, boolean isNotNull) { + return getQuery (null, isNotNull, columnName); + } + + private String getQuery (String allColumns, boolean isNotNull, String column) { + StringBuffer query = new StringBuffer(allColumns != null ? table.getSelectQuery("*") : table.getSelectQuery(column)); + query.append(isNotNull ? " where " + column + " is not null;" : " where " + column + " is null;"); + return query.toString(); + } +} + From 9a6aa9a952a6d293ccae8a469b50db511ed74d15 Mon Sep 17 00:00:00 2001 From: YashpalThakur <36757209+YashpalThakur@users.noreply.github.com> Date: Mon, 16 Apr 2018 13:22:00 +0530 Subject: [PATCH 089/129] Deleted this file to add new file with changes --- .../jdbc/h2/JdbcToArrowTestDatatypes.java | 319 ------------------ 1 file changed, 319 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java deleted file mode 100644 index 92f8b858c68..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestDatatypes.java +++ /dev/null @@ -1,319 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.adapter.jdbc.h2; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import java.util.Arrays; -import java.util.Collection; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.adapter.jdbc.Table; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeStampVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; - -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; - -@RunWith(Parameterized.class) -public class JdbcToArrowTestDatatypes { - private static Connection conn = null; - private static Table table; - private int [] intValues; - private int [] boolValues; - private BigDecimal [] decimalValues; - private double [] doubleValues; - private int [] tinyIntValues; - private int [] smallIntValues; - private int [] bigIntValues; - private long [] timeValues; - private long [] dateValues; - private long [] timestampValues; - private float [] realValues; - private byte [][] byteValues; - private byte [][] varCharValues; - private byte [][] charValues; - private byte [][] clobValues; - - public JdbcToArrowTestDatatypes (Table data) { - super(); - this.intValues = data.getInts(); - this.boolValues = data.getBooleans(); - this.decimalValues = data.getDecimals(); - this.doubleValues = data.getDoubles(); - this.tinyIntValues = data.getTinyInts(); - this.smallIntValues = data.getSmallInts(); - this.bigIntValues = data.getBigInts(); - this.timeValues = data.getTimes(); - this.dateValues = data.getDates(); - this.timestampValues = data.getTimestamps(); - this.realValues = data.getReals(); - this.byteValues = data.getHexStringAsByte(); - this.varCharValues = data.getVarCharAsByte(); - this.charValues = data.getCharAsByte(); - this.clobValues = data.getClobAsByte(); - - } - - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException { - setUp(); - return Arrays.asList(new Object[][]{{table}}); - } - - @Test - public void testDBValues() { - try { - sqlToArrowTestInt(); - sqlToArrowTestBool(); - sqlToArrowTestTinyInts(); - sqlToArrowTestSmallInts(); - sqlToArrowTestBigInts(); - sqlToArrowTestBigDecimals(); - sqlToArrowTestDoubles(); - sqlToArrowTestRealValues(); - sqlToArrowTestTimeValues(); - sqlToArrowTestDateValues(); - sqlToArrowTestTimestampValues(); - sqlToArrowTestByteValues(); - sqlToArrowTestVarCharValues(); - sqlToArrowTestCharValues(); - sqlToArrowTestBlobValues(); - sqlToArrowTestClobValues(); - sqlToArrowTestBits(); - sqlToArrowTestNullValues(); - } catch (SQLException sqe) { - sqe.printStackTrace(); - } - } - - public void sqlToArrowTestInt() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1", true), new RootAllocator(Integer.MAX_VALUE))){ - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - } - } - - public void sqlToArrowTestBool() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2", true), new RootAllocator(Integer.MAX_VALUE))){ - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); - } - } - - public void sqlToArrowTestTinyInts() throws SQLException { - try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3", true), new RootAllocator(Integer.MAX_VALUE))){ - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); - } - } - - public void sqlToArrowTestSmallInts() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4", true), new RootAllocator(Integer.MAX_VALUE))){ - assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); - } - } - - public void sqlToArrowTestBigInts() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIGINT_FIELD5", true), new RootAllocator(Integer.MAX_VALUE))){ - assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); - } - } - - public void sqlToArrowTestBigDecimals() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DECIMAL_FIELD6", true), new RootAllocator(Integer.MAX_VALUE))){ - assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); - } - } - - public void sqlToArrowTestDoubles() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DOUBLE_FIELD7", true), new RootAllocator(Integer.MAX_VALUE))){ - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); - } - } - - public void sqlToArrowTestRealValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("REAL_FIELD8", true), new RootAllocator(Integer.MAX_VALUE))){ - assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); - } - } - - public void sqlToArrowTestTimeValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIME_FIELD9", true), new RootAllocator(Integer.MAX_VALUE))){ - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); - } - } - - public void sqlToArrowTestDateValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DATE_FIELD10", true), new RootAllocator(Integer.MAX_VALUE))){ - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); - } - } - - public void sqlToArrowTestTimestampValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIMESTAMP_FIELD11", true), new RootAllocator(Integer.MAX_VALUE))){ - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); - } - } - - public void sqlToArrowTestByteValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BINARY_FIELD12", true), new RootAllocator(Integer.MAX_VALUE))){ - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); - - } - } - - public void sqlToArrowTestVarCharValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("VARCHAR_FIELD13", true), new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); - } - } - - public void sqlToArrowTestBlobValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BLOB_FIELD14", true), new RootAllocator(Integer.MAX_VALUE))){ - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); - } - } - - public void sqlToArrowTestClobValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CLOB_FIELD15", true), new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); - } - } - - public void sqlToArrowTestCharValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CHAR_FIELD16", true), new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); - } - } - - public void sqlToArrowTestBits() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIT_FIELD17", true), new RootAllocator(Integer.MAX_VALUE))){ - assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); - } - } - - public void sqlToArrowTestNullValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("all", false, "INT_FIELD1"), new RootAllocator(Integer.MAX_VALUE))){ - assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); - assertNullValues((BitVector)root.getVector("BOOL_FIELD2"), 5); - assertNullValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 5); - assertNullValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 5); - assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); - assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); - assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); - assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); - assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); - assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); - assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); - assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); - assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); - assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); - assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); - assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); - assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); - } - } - - @Before - public void createTestData() throws SQLException{ - try (Statement stmt = conn.createStatement();){ - stmt.executeUpdate(table.getCreate()); - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - } - } - - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } - - private static void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - table = getTable ("h2/test1_all_datatypes_h2.yml"); - } - - private static Table getTable (String ymlFilePath) { - Table table = null; - try { - table = new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowTestDatatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } catch (JsonMappingException jme) { - jme.printStackTrace(); - } catch (JsonParseException jpe) { - jpe.printStackTrace(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - return table; - } - - private String getQuery (String columnName, boolean isNotNull) { - return getQuery (null, isNotNull, columnName); - } - - private String getQuery (String allColumns, boolean isNotNull, String column) { - StringBuffer query = new StringBuffer(allColumns != null ? table.getSelectQuery("*") : table.getSelectQuery(column)); - query.append(isNotNull ? " where " + column + " is not null;" : " where " + column + " is null;"); - return query.toString(); - } -} - From d1b0992b440a1d585bd67a5f4711c38a54c46eb5 Mon Sep 17 00:00:00 2001 From: yashpal Date: Mon, 16 Apr 2018 05:58:11 -0400 Subject: [PATCH 090/129] files commited to merge changes from upstream --- .../arrow/adapter/jdbc/JdbcToArrow.java | 81 ++++++++++++++++--- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 60 ++++++-------- 2 files changed, 95 insertions(+), 46 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 650cc679c45..e835eeb2ef1 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -28,6 +28,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.ResultSet; +import java.util.Calendar; /** * Utility class to convert JDBC objects to columnar Arrow format objects. @@ -59,12 +60,17 @@ * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary * + * TODO: At this time, SQL Data type java.sql.Types.ARRAY is still not supported. + * * @since 0.10.0 */ public class JdbcToArrow { /** * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * This method uses the default Calendar instance with default TimeZone and Locale as returned by the JVM. + * If you wish to use specific TimeZone or Locale for any Date, Time and Timestamp datasets, you may want use + * overloaded API that taken Calendar object instance. * * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed * the connection object it's the responsibility of the caller to close or return the connection to the pool. @@ -78,23 +84,74 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + return sqlToArrow(connection, query, allocator, Calendar.getInstance()); + } + + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. + */ + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), allocator); + return sqlToArrow(stmt.executeQuery(query), allocator, calendar); } } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. This method + * uses the default RootAllocator and Calendar object. * * @param resultSet * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws Exception + * @throws SQLException */ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + return sqlToArrow(resultSet, Calendar.getInstance()); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param allocator Memory allocator + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + + return sqlToArrow(resultSet, allocator, Calendar.getInstance()); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator); + VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); rootAllocator.close(); return root; } @@ -103,17 +160,19 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. * * @param resultSet - * @param allocator Memory allocator + * @param allocator Memory allocator to use. + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws Exception + * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(allocator, "Root Allocator object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData()), allocator); - JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root); + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), calendar), allocator); + JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, calendar); return root; } -} +} \ No newline at end of file diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 0a1867117cf..e5ac699ce1d 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -36,7 +36,6 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.impl.NullableVarCharHolderReaderImpl; import org.apache.arrow.vector.holders.NullableBigIntHolder; import org.apache.arrow.vector.holders.NullableBitHolder; import org.apache.arrow.vector.holders.NullableDateMilliHolder; @@ -70,6 +69,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; @@ -119,9 +119,10 @@ public class JdbcToArrowUtils { * @return {@link Schema} * @throws SQLException */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLException { + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); List fields = new ArrayList<>(); int columnCount = rsmd.getColumnCount(); @@ -172,8 +173,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd) throws SQLExcepti fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); break; case Types.TIMESTAMP: - // TODO Need to handle timezone - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)), null)); + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar.getTimeZone().getID())), null)); break; case Types.BINARY: case Types.VARBINARY: @@ -220,10 +220,11 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * @param root Arrow {@link VectorSchemaRoot} object to populate * @throws SQLException */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throws SQLException { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); ResultSetMetaData rsmd = rs.getMetaData(); int columnCount = rsmd.getColumnCount(); @@ -232,9 +233,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw int rowCount = 0; while (rs.next()) { - // for each column get the value based on the type - - // need to change this to build Java lists and then build Arrow vectors for (int i = 1; i <= columnCount; i++) { String columnName = rsmd.getColumnName(i); switch (rsmd.getColumnType(i)) { @@ -284,34 +282,34 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root) throw break; case Types.DATE: updateVector((DateMilliVector) root.getVector(columnName), - rs.getDate(i), !rs.wasNull(), rowCount); + rs.getDate(i, calendar), !rs.wasNull(), rowCount); break; case Types.TIME: updateVector((TimeMilliVector) root.getVector(columnName), - rs.getTime(i), !rs.wasNull(), rowCount); + rs.getTime(i, calendar), !rs.wasNull(), rowCount); break; case Types.TIMESTAMP: // TODO: Need to handle precision such as milli, micro, nano updateVector((TimeStampVector)root.getVector(columnName), - rs.getTimestamp(i), !rs.wasNull(), rowCount); + rs.getTimestamp(i, calendar), !rs.wasNull(), rowCount); break; case Types.BINARY: case Types.VARBINARY: case Types.LONGVARBINARY: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBytes(i), rowCount); + rs.getBytes(i), !rs.wasNull(), rowCount); break; case Types.ARRAY: // TODO Need to handle this type -// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); break; case Types.CLOB: updateVector((VarCharVector)root.getVector(columnName), - rs.getClob(i), rowCount); + rs.getClob(i), !rs.wasNull(), rowCount); break; case Types.BLOB: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBlob(i), rowCount); + rs.getBlob(i), !rs.wasNull(), rowCount); break; default: @@ -412,26 +410,20 @@ private static void updateVector(VarCharVector varcharVector, String value, bool NullableVarCharHolder holder = new NullableVarCharHolder(); holder.isSet = isNonNull? 1: 0; varcharVector.setIndexDefined(rowCount); - if (isNonNull) { byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - int bytesLength = bytes.length; - - holder.buffer = varcharVector.getAllocator().buffer(bytesLength); + holder.buffer = varcharVector.getAllocator().buffer(bytes.length); + holder.buffer.setBytes(0, bytes, 0, bytes.length); holder.start = 0; - holder.end = bytesLength; - holder.buffer.setBytes(0, bytes, 0, bytesLength); + holder.end = bytes.length; } else { holder.buffer = varcharVector.getAllocator().buffer(0); } - varcharVector.setSafe(rowCount, holder); - varcharVector.setValueCount(rowCount + 1); - + varcharVector.setValueCount(rowCount + 1); } private static void updateVector(DateMilliVector dateMilliVector, Date date, boolean isNonNull, int rowCount) { - //TODO: Need to handle Timezone NullableDateMilliHolder holder = new NullableDateMilliHolder(); holder.isSet = isNonNull? 1: 0; if (isNonNull) { @@ -442,10 +434,9 @@ private static void updateVector(DateMilliVector dateMilliVector, Date date, boo } private static void updateVector(TimeMilliVector timeMilliVector, Time time, boolean isNonNull, int rowCount) { - //TODO: Need to handle Timezone NullableTimeMilliHolder holder = new NullableTimeMilliHolder(); holder.isSet = isNonNull? 1: 0; - if (isNonNull) { + if (isNonNull && time != null) { holder.value = (int)time.getTime(); } timeMilliVector.setSafe(rowCount, holder); @@ -453,7 +444,6 @@ private static void updateVector(TimeMilliVector timeMilliVector, Time time, boo } private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, boolean isNonNull, int rowCount) { - //TODO: Need to handle timezone //TODO: Need to handle precision such as milli, micro, nano timeStampVector.setValueCount(rowCount + 1); if (timestamp != null) { @@ -463,9 +453,9 @@ private static void updateVector(TimeStampVector timeStampVector, Timestamp time } } - private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, int rowCount) { + private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, boolean isNonNull, int rowCount) { varBinaryVector.setValueCount(rowCount + 1); - if (bytes != null) { + if (isNonNull && bytes != null) { varBinaryVector.setIndexDefined(rowCount); varBinaryVector.setValueLengthSafe(rowCount, bytes.length); varBinaryVector.setSafe(rowCount, bytes); @@ -474,9 +464,9 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, } } - private static void updateVector(VarCharVector varcharVector, Clob clob, int rowCount) throws SQLException { + private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException { varcharVector.setValueCount(rowCount + 1); - if (clob != null) { + if (isNonNull && clob != null) { int length = (int) clob.length(); String value = clob.getSubString(1, length); if (value != null) { @@ -491,9 +481,9 @@ private static void updateVector(VarCharVector varcharVector, Clob clob, int row } } - private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int rowCount) throws SQLException { + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException { varBinaryVector.setValueCount(rowCount + 1); - if (blob != null) { + if (isNonNull && blob != null) { byte[] data = blob.getBytes(0, (int) blob.length()); varBinaryVector.setIndexDefined(rowCount); varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); @@ -503,4 +493,4 @@ private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, int } } -} +} \ No newline at end of file From b95e87807ee0e2fa315fa350150d5cfaa8fc33aa Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 19 Apr 2018 06:28:02 -0400 Subject: [PATCH 091/129] file committed for timezone and null value check review comment implementation --- .../resources/h2/test1_all_datatypes_h2.yml | 280 +++++++++++------- 1 file changed, 165 insertions(+), 115 deletions(-) diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 0a8ab5ae89f..7e04bdf5e78 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -9,98 +9,6 @@ #OF ANY KIND, either express or implied. See the License for the specific #language governing permissions and limitations under the License. -name: 'table1' - -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' - -data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - -query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, - time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' - -drop: 'DROP table table1;' - ints: - '101' - '101' @@ -151,7 +59,7 @@ tinyInts: - '45' - '45' - '45' - + smallInts: - '12000' - '12000' @@ -185,7 +93,7 @@ bigInts: - '92233720' - '92233720' - '92233720' - + reals: - '56478356785.345f' - '56478356785.345f' @@ -202,7 +110,7 @@ reals: - '56478356785.345f' - '56478356785.345f' - '56478356785.345f' - + decimals: - '17345667789.23' - '17345667789.23' @@ -304,7 +212,7 @@ bytes: - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - + varchars: - 'some text that needs to be converted to varchar' - 'some text that needs to be converted to varchar' @@ -356,22 +264,164 @@ chars: - 'some char text' - 'some char text' -selectQuereis: - - 'select int_field1 from table1' - - 'select bool_field2 from table1' - - 'select tinyint_field3 from table1' - - 'select smallint_field4 from table1' - - 'select bigint_field5 from table1' - - 'select decimal_field6 from table1' - - 'select double_field7 from table1' - - 'select real_field8 from table1' - - 'select time_field9 from table1' - - 'select date_field10 from table1' - - 'select timestamp_field11 from table1' - - 'select binary_field12 from table1' - - 'select varchar_field13 from table1' - - 'select blob_field14 from table1' - - 'select clob_field15 from table1' - - 'select char_field16 from table1' - - 'select bit_field17 from table1' - - 'select * from table1' \ No newline at end of file +pstTime: + - '56735000' + - '56735000' + - '56735000' + - '56735000' + - '56735000' + +estTime: + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + +gmtTime: + - '27935000' + - '27935000' + - '27935000' + - '27935000' + - '27935000' + +pstDate: + - '1518336000000' + - '1518336000000' + - '1518336000000' + - '1518336000000' + - '1518336000000' + +estDate: + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + - '1518325200000' + +gmtDate: + - '1518307200000' + - '1518307200000' + - '1518307200000' + - '1518307200000' + - '1518307200000' + +pstTimestamp: + - '1518450335000' + - '1518450335000' + - '1518450335000' + - '1518450335000' + - '1518450335000' + +estTimestamp: + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + +gmtTimestamp: + - '1518421535000' + - '1518421535000' + - '1518421535000' + - '1518421535000' + - '1518421535000' + +allColumns: 'int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17' + +name: 'table1' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + + - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), + PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', + ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + +query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, + time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' + +drop: 'DROP table table1;' \ No newline at end of file From da1227d27748d8a9aa1cc5096370cf370ab1d0dd Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 19 Apr 2018 06:28:28 -0400 Subject: [PATCH 092/129] file committed for timezone and null value check review comment implementation --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index e3deb98f78d..64e153bf562 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -52,7 +52,7 @@ public static void assertIntVectorValues(IntVector intVector, int rowCount, int assertEquals(values[j], intVector.get(j)); } } - + public static void assertBitBooleanVectorValues(BitVector bitVector, int rowCount, int[] values){ assertEquals(rowCount, bitVector.getValueCount()); @@ -135,26 +135,18 @@ public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, } public static void assertVarBinaryVectorValues (VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { - try { - assertEquals(rowCount, varBinaryVector.getValueCount()); - - for(int j = 0; j < varBinaryVector.getValueCount(); j++){ - assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varBinaryVector.get(j))); - } - } catch (AssertionError ae) { - ae.printStackTrace(); + assertEquals(rowCount, varBinaryVector.getValueCount()); + + for(int j = 0; j < varBinaryVector.getValueCount(); j++){ + assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varBinaryVector.get(j))); } } public static void assertVarcharVectorValues(VarCharVector varCharVector, int rowCount, byte[][] values) { - try { - assertEquals(rowCount, varCharVector.getValueCount()); - - for(int j = 0; j < varCharVector.getValueCount(); j++){ - assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varCharVector.get(j))); - } - } catch (AssertionError ae) { - ae.printStackTrace(); + assertEquals(rowCount, varCharVector.getValueCount()); + + for(int j = 0; j < varCharVector.getValueCount(); j++){ + assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varCharVector.get(j))); } } From 857a4b463a4e40d80a7102df62f1ebc9e461a45c Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 19 Apr 2018 06:28:56 -0400 Subject: [PATCH 093/129] file committed for timezone and null value check review comment implementation --- .../org/apache/arrow/adapter/jdbc/Table.java | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index ba7cae2632b..7c56fac4ce6 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -54,9 +54,18 @@ public class Table { private float [] reals; private double [] doubles; private BigDecimal [] decimals; - private String [] selectQuereis; - - public Table() { + private String allColumns; + private long [] pstTime; + private long [] estTime; + private long [] gmtTime; + private long [] pstDate; + private long [] estDate; + private long [] gmtDate; + private long [] pstTimestamp; + private long [] estTimestamp; + private long [] gmtTimestamp; + + public Table() { } public String getName() { @@ -179,19 +188,12 @@ public float[] getReals() { public void setReals(float[] reals) { this.reals = reals; } - public String[] getSelectQuereis() { - return selectQuereis; + public String getAllColumns() { + return allColumns; } - public void setSelectQuereis(String[] selectQuereis) { - this.selectQuereis = selectQuereis; - } - - public String getSelectQuery(String columnName) { - String queryString = "select " + columnName + " from"; - String query = Arrays.stream(selectQuereis).parallel().filter(q -> q.toUpperCase().contains(queryString.toUpperCase())).findFirst().get(); - return query; + public void setAllColumns(String allColumns) { + this.allColumns = allColumns; } - public byte [][] getHexStringAsByte () { return getHexToByteArray (bytes); } @@ -204,6 +206,60 @@ public String getSelectQuery(String columnName) { public byte [][] getVarCharAsByte () { return getByteArray (varchars); } + public long[] getPstTime() { + return pstTime; + } + public void setPstTime(long[] pstTime) { + this.pstTime = pstTime; + } + public long[] getEstTime() { + return estTime; + } + public void setEstTime(long[] estTime) { + this.estTime = estTime; + } + public long[] getGmtTime() { + return gmtTime; + } + public void setGmtTime(long[] gmtTime) { + this.gmtTime = gmtTime; + } + public long[] getPstDate() { + return pstDate; + } + public void setPstDate(long[] pstDate) { + this.pstDate = pstDate; + } + public long[] getEstDate() { + return estDate; + } + public void setEstDate(long[] estDate) { + this.estDate = estDate; + } + public long[] getGmtDate() { + return gmtDate; + } + public void setGmtDate(long[] gmtDate) { + this.gmtDate = gmtDate; + } + public long[] getPstTimestamp() { + return pstTimestamp; + } + public void setPstTimestamp(long[] pstTimestamp) { + this.pstTimestamp = pstTimestamp; + } + public long[] getEstTimestamp() { + return estTimestamp; + } + public void setEstTimestamp(long[] estTimestamp) { + this.estTimestamp = estTimestamp; + } + public long[] getGmtTimestamp() { + return gmtTimestamp; + } + public void setGmtTimestamp(long[] gmtTimestamp) { + this.gmtTimestamp = gmtTimestamp; + } private byte [][] getByteArray (String [] data) { byte [][] byteArr = new byte [data.length][]; @@ -212,8 +268,7 @@ public String getSelectQuery(String columnName) { byteArr [i] = data [i].getBytes(StandardCharsets.UTF_8); } return byteArr; - } - + } private byte [][] getHexToByteArray (String [] data){ byte [][] byteArr = new byte [data.length][]; @@ -222,8 +277,7 @@ public String getSelectQuery(String columnName) { } return byteArr; } - - private static byte[] hexStringToByteArray(String s) { + private static byte[] hexStringToByteArray(String s) { int len = s.length(); byte[] data = new byte[len / 2]; for (int i = 0; i < len; i += 2) { From 6b0b00e01c6ef388af092b8615dc5d9e4a6114bb Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 19 Apr 2018 06:29:30 -0400 Subject: [PATCH 094/129] file committed for timezone and null value check review comment implementation --- .../jdbc/h2/JdbcToArrowTestH2Datatypes.java | 213 +++++++++++++----- 1 file changed, 156 insertions(+), 57 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java index 55fdbe5e71c..ee640988430 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java @@ -24,9 +24,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; +import java.util.TimeZone; + import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -51,6 +55,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.time.ZoneId; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; @@ -86,7 +91,16 @@ public class JdbcToArrowTestH2Datatypes { private byte [][] varCharValues; private byte [][] charValues; private byte [][] clobValues; - + private long [] pstTimeValues; + private long [] estTimeValues; + private long [] gmtTimeValues; + private long [] pstDateValues; + private long [] estDateValues; + private long [] gmtDateValues; + private long [] pstTimestampValues; + private long [] estTimestampValues; + private long [] gmtTimestampValues; + public JdbcToArrowTestH2Datatypes (Table data) { super(); this.intValues = data.getInts(); @@ -104,19 +118,28 @@ public JdbcToArrowTestH2Datatypes (Table data) { this.varCharValues = data.getVarCharAsByte(); this.charValues = data.getCharAsByte(); this.clobValues = data.getClobAsByte(); - + this.pstTimeValues = data.getPstTime(); + this.estTimeValues = data.getEstTime(); + this.gmtTimeValues = data.getGmtTime(); + this.pstDateValues = data.getPstDate(); + this.estDateValues = data.getEstDate(); + this.gmtDateValues = data.getGmtDate(); + this.pstTimestampValues = data.getPstTimestamp(); + this.estTimestampValues = data.getEstTimestamp() ; + this.gmtTimestampValues = data.getGmtTimestamp(); + } - + @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException { + public static Collection getTestData() throws SQLException, ClassNotFoundException, JsonMappingException, JsonParseException, IOException { setUp(); return Arrays.asList(new Object[][]{{table}}); } - + @Test public void testDBValues() { try { - sqlToArrowTestInt(); + sqlToArrowTestInt(); sqlToArrowTestBool(); sqlToArrowTestTinyInts(); sqlToArrowTestSmallInts(); @@ -134,116 +157,140 @@ public void testDBValues() { sqlToArrowTestClobValues(); sqlToArrowTestBits(); sqlToArrowTestNullValues(); + sqlToArrowTestValuesWithPSTTimeZone(); + sqlToArrowTestValuesWithESTTimeZone(); + sqlToArrowTestValuesWithGMTTimeZone(); + sqlToArrowTestSelectedColumnsNullValues(); } catch (SQLException sqe) { sqe.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); } } - + public void sqlToArrowTestInt() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1 = 101", true, "INT_FIELD1"), + new RootAllocator(Integer.MAX_VALUE))){ assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); } } - + public void sqlToArrowTestBool() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2 = 1", true, "BOOL_FIELD2"), + new RootAllocator(Integer.MAX_VALUE))){ assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); } } - + public void sqlToArrowTestTinyInts() throws SQLException { - try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3", true), new RootAllocator(Integer.MAX_VALUE))){ + try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3 = 45", true, "TINYINT_FIELD3"), + new RootAllocator(Integer.MAX_VALUE))){ assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); } } public void sqlToArrowTestSmallInts() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4 = 12000", true, "SMALLINT_FIELD4"), + new RootAllocator(Integer.MAX_VALUE))){ assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); } } public void sqlToArrowTestBigInts() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIGINT_FIELD5", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIGINT_FIELD5"), + new RootAllocator(Integer.MAX_VALUE))){ assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); } } public void sqlToArrowTestBigDecimals() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DECIMAL_FIELD6", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DECIMAL_FIELD6"), + new RootAllocator(Integer.MAX_VALUE))){ assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); } } public void sqlToArrowTestDoubles() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DOUBLE_FIELD7", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DOUBLE_FIELD7"), + new RootAllocator(Integer.MAX_VALUE))){ assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); } } - + public void sqlToArrowTestRealValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("REAL_FIELD8", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "REAL_FIELD8"), + new RootAllocator(Integer.MAX_VALUE))){ assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); } } - + public void sqlToArrowTestTimeValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIME_FIELD9", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIME_FIELD9"), + new RootAllocator(Integer.MAX_VALUE))){ assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); } } public void sqlToArrowTestDateValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("DATE_FIELD10", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DATE_FIELD10"), + new RootAllocator(Integer.MAX_VALUE))){ assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); } - } + } public void sqlToArrowTestTimestampValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TIMESTAMP_FIELD11", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIMESTAMP_FIELD11"), + new RootAllocator(Integer.MAX_VALUE))){ assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); } } - + public void sqlToArrowTestByteValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BINARY_FIELD12", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BINARY_FIELD12"), + new RootAllocator(Integer.MAX_VALUE))){ assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); } } - + public void sqlToArrowTestVarCharValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("VARCHAR_FIELD13", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "VARCHAR_FIELD13"), + new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); } } public void sqlToArrowTestBlobValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BLOB_FIELD14", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BLOB_FIELD14"), + new RootAllocator(Integer.MAX_VALUE))){ assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); } } public void sqlToArrowTestClobValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CLOB_FIELD15", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CLOB_FIELD15"), + new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); } } - + public void sqlToArrowTestCharValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("CHAR_FIELD16", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CHAR_FIELD16"), + new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); } } public void sqlToArrowTestBits() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BIT_FIELD17", true), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIT_FIELD17"), + new RootAllocator(Integer.MAX_VALUE))){ assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); } } public void sqlToArrowTestNullValues() throws SQLException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("all", false, "INT_FIELD1"), new RootAllocator(Integer.MAX_VALUE))){ + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, + getQuery (false, table.getAllColumns()), new RootAllocator(Integer.MAX_VALUE))){ assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); assertNullValues((BitVector)root.getVector("BOOL_FIELD2"), 5); assertNullValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 5); @@ -259,7 +306,59 @@ public void sqlToArrowTestNullValues() throws SQLException { assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); - assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); + assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); + assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); + } + } + + public void sqlToArrowTestValuesWithPSTTimeZone() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, + getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("PST")))){ + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), pstTimeValues.length, pstTimeValues); + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), pstDateValues.length, pstDateValues); + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), pstTimestampValues.length, + pstTimestampValues); + } + } + + public void sqlToArrowTestValuesWithESTTimeZone() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, + getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("EST")))){ + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), estTimeValues.length, estTimeValues); + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), estDateValues.length, estDateValues); + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), estTimestampValues.length, + estTimestampValues); + } + } + + public void sqlToArrowTestValuesWithGMTTimeZone() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, + getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("GMT")))){ + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), gmtTimeValues.length, gmtTimeValues); + assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), gmtDateValues.length, gmtDateValues); + assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), gmtTimestampValues.length, + gmtTimestampValues); + } + } + + public void sqlToArrowTestSelectedColumnsNullValues() throws SQLException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("INT_FIELD1 = 102", true, table.getAllColumns()), + new RootAllocator(Integer.MAX_VALUE))){ + assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); + assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); + assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); + assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); + assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); + assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); + assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); + assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); + assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); + assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); + assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); + assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); } } @@ -277,43 +376,43 @@ public void createTestData() throws SQLException{ @After public void destroy() throws SQLException { if (conn != null) { - conn.close(); + conn.close(); conn = null; } } - private static void setUp() throws SQLException, ClassNotFoundException { + private static void setUp() throws SQLException, ClassNotFoundException, JsonMappingException, JsonParseException, IOException { String url = "jdbc:h2:mem:JdbcToArrowTest"; String driver = "org.h2.Driver"; Class.forName(driver); conn = DriverManager.getConnection(url); table = getTable ("h2/test1_all_datatypes_h2.yml"); } - - private static Table getTable (String ymlFilePath) { - Table table = null; - try { - table = new ObjectMapper(new YAMLFactory()).readValue( + + private static Table getTable (String ymlFilePath) throws JsonMappingException, JsonParseException, IOException { + return new ObjectMapper(new YAMLFactory()).readValue( JdbcToArrowTestH2Datatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); - } catch (JsonMappingException jme) { - jme.printStackTrace(); - } catch (JsonParseException jpe) { - jpe.printStackTrace(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - return table; } + + private String getQuery (boolean isNotNull, String... columns) { + return getQuery("", isNotNull, columns); - private String getQuery (String columnName, boolean isNotNull) { - return getQuery (null, isNotNull, columnName); - } + } - private String getQuery (String allColumns, boolean isNotNull, String column) { - StringBuffer query = new StringBuffer(allColumns != null ? table.getSelectQuery("*") : table.getSelectQuery(column)); - query.append(isNotNull ? " where " + column + " is not null;" : " where " + column + " is null;"); - return query.toString(); - } + private String getQuery (String whereClause, boolean isNotNull, String... columns) { + return getQuery( whereClause, isNotNull, 0, columns); + + } + + private String getQuery(String whereClause, boolean isNotNull, int index, String... columns) { + StringBuffer query = new StringBuffer(String.format("select %s from %s ", + columns.length > 1 ? String.join(",", columns) : columns[index], table.getName())); + query.append(whereClause != null && !whereClause.isEmpty() ? " where " + whereClause + " and " : " where "); + columns = columns.length == 1 && columns[0].contains(",") ? columns[0].split(",") : columns; + query.append(isNotNull ? columns[index] + " is not null;" : columns[index] + " is null;"); + + return query.toString(); + } } From 7cc649c84cd0129cd242af7b7ae75ccdbd53f6d5 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 24 Apr 2018 02:45:31 -0400 Subject: [PATCH 095/129] File committed after changing to StringBuilder --- .../arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java index ee640988430..a36be61c55a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java @@ -30,7 +30,6 @@ import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; -import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -55,7 +54,6 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.time.ZoneId; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; @@ -406,7 +404,7 @@ private String getQuery (String whereClause, boolean isNotNull, String... column } private String getQuery(String whereClause, boolean isNotNull, int index, String... columns) { - StringBuffer query = new StringBuffer(String.format("select %s from %s ", + StringBuilder query = new StringBuilder(String.format("select %s from %s ", columns.length > 1 ? String.join(",", columns) : columns[index], table.getName())); query.append(whereClause != null && !whereClause.isEmpty() ? " where " + whereClause + " and " : " where "); columns = columns.length == 1 && columns[0].contains(",") ? columns[0].split(",") : columns; From 75d5d1fd3d83206dca1ab4dc88c6520410250734 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 24 Apr 2018 17:44:22 -0700 Subject: [PATCH 096/129] Merged pull request code related to NULL checks. Fixes as per code review suggestions. --- java/adapter/jdbc/pom.xml | 12 ++++++++++++ .../apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 7 ------- .../arrow/adapter/jdbc/h2/JdbcToArrowTest.java | 6 +++--- .../src/test/resources/h2/test1_all_datatypes_h2.yml | 10 ---------- .../jdbc/src/test/resources/h2/test1_blob_h2.yml | 2 +- .../jdbc/src/test/resources/h2/test1_clob_h2.yml | 2 +- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index c67cab13589..5a879247746 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -68,6 +68,18 @@ ${dep.jackson.version} test + + com.fasterxml.jackson.core + jackson-core + ${dep.jackson.version} + test + + + com.fasterxml.jackson.core + jackson-annotations + ${dep.jackson.version} + test + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 5e3ad4c79b8..e5ac699ce1d 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -342,13 +342,6 @@ private static void updateVector(TinyIntVector tinyIntVector, int value, boolean tinyIntVector.setValueCount(rowCount + 1); } - private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { - NullableSmallIntHolder holder = new NullableSmallIntHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = (short) value; - } - smallIntVector.setSafe(rowCount, value); private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { NullableSmallIntHolder holder = new NullableSmallIntHolder(); holder.isSet = isNonNull? 1: 0; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index fd98ec61f7a..f7ef85068ff 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -293,7 +293,7 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, ints); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, table.getInts()); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 @@ -346,8 +346,8 @@ public void sqlToArrowTestAllDataTypes() throws Exception { assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 15, times); long[] dates = { - 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, - 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l, 1518325200000l + 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, + 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l }; assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), 15, dates); diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 7e04bdf5e78..ef75f323f4d 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -410,16 +410,6 @@ data: PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml index d360773c55f..0416a2e7fd6 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml @@ -62,6 +62,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select blob_field14 from table1;' drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml index d360773c55f..1518e8c1ed5 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml @@ -62,6 +62,6 @@ data: ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' -query: 'select int_field1 from table1;' +query: 'select CLOB_FIELD15 from table1;' drop: 'DROP table table1;' \ No newline at end of file From c2ba1ee773ee1dfd0c43f85bdaf6e69f3d181853 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Tue, 24 Apr 2018 18:03:18 -0700 Subject: [PATCH 097/129] Removed unused imports. --- .../src/test/java/org/apache/arrow/adapter/jdbc/Table.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 7c56fac4ce6..76940ba04a9 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -20,11 +20,6 @@ import java.math.BigDecimal; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; From e2ba906df433636c64519ed6a15b9edfa88cbe96 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 25 Apr 2018 12:58:19 -0700 Subject: [PATCH 098/129] Updated Blob, Clob and Binary objects to use streaming approach while populating the Arrow Vector objects. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 13 ++-- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 77 ++++++++++++++----- .../jdbc/h2/JdbcToArrowTestH2Datatypes.java | 65 ++++++++++------ 3 files changed, 104 insertions(+), 51 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index db013e92682..0bdcc007e3e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; +import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -79,7 +80,7 @@ public class JdbcToArrow { * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException, IOException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); @@ -98,7 +99,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException { + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); @@ -117,7 +118,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); return sqlToArrow(resultSet, Calendar.getInstance()); @@ -131,7 +132,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); @@ -146,7 +147,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator all * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(calendar, "Calendar object can not be null"); @@ -165,7 +166,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); Preconditions.checkNotNull(calendar, "Calendar object can not be null"); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index e5ac699ce1d..d6adc3c9223 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -48,6 +49,8 @@ import org.apache.arrow.vector.holders.NullableTinyIntHolder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; +import org.apache.arrow.vector.holders.VarBinaryHolder; +import org.apache.arrow.vector.holders.VarCharHolder; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -56,6 +59,10 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.DecimalUtility; +import java.io.ByteArrayInputStream; +import java.io.IOError; +import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; @@ -84,6 +91,7 @@ public class JdbcToArrowUtils { private static final int DEFAULT_BUFFER_SIZE = 256; + private static final int DEFAULT_STREAM_BUFFER_SIZE = 1024; /** * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. @@ -220,7 +228,7 @@ private static void allocateVectors(VectorSchemaRoot root, int size) { * @param root Arrow {@link VectorSchemaRoot} object to populate * @throws SQLException */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException { + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); @@ -297,7 +305,8 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calen case Types.VARBINARY: case Types.LONGVARBINARY: updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBytes(i), !rs.wasNull(), rowCount); +// rs.getBytes(i), !rs.wasNull(), rowCount); + rs.getBinaryStream(i), !rs.wasNull(), rowCount); break; case Types.ARRAY: // TODO Need to handle this type @@ -464,33 +473,59 @@ private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, } } - private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException { + private static void updateVector(VarBinaryVector varBinaryVector, InputStream is, boolean isNonNull, int rowCount) throws IOException { + varBinaryVector.setValueCount(rowCount + 1); + if (isNonNull && is != null) { + VarBinaryHolder holder = new VarBinaryHolder(); + ArrowBuf arrowBuf = varBinaryVector.getDataBuffer(); + holder.start = 0; + byte[] bytes = new byte[DEFAULT_STREAM_BUFFER_SIZE]; + int total = 0; + while (true) { + int read = is.read(bytes, 0, DEFAULT_STREAM_BUFFER_SIZE); + if (read == -1) { + break; + } + total += read; + arrowBuf.setBytes(!arrowBuf.hasArray()? 0: arrowBuf.arrayOffset(), new ByteArrayInputStream(bytes, 0, read), read); + } + holder.end = total; + holder.buffer = arrowBuf; + varBinaryVector.set(rowCount, holder); + varBinaryVector.setIndexDefined(rowCount); + } else { + varBinaryVector.setNull(rowCount); + } + } + + private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException, IOException { varcharVector.setValueCount(rowCount + 1); if (isNonNull && clob != null) { - int length = (int) clob.length(); - String value = clob.getSubString(1, length); - if (value != null) { - varcharVector.setIndexDefined(rowCount); - varcharVector.setValueLengthSafe(rowCount, length); - varcharVector.setSafe(rowCount, value.getBytes(StandardCharsets.UTF_8), 0, length); - } else { - varcharVector.setNull(rowCount); + VarCharHolder holder = new VarCharHolder(); + ArrowBuf arrowBuf = varcharVector.getDataBuffer(); + InputStream is = clob.getAsciiStream(); + holder.start = 0; + byte[] bytes = new byte[DEFAULT_STREAM_BUFFER_SIZE]; + int total = 0; + while (true) { + int read = is.read(bytes, 0, DEFAULT_STREAM_BUFFER_SIZE); + if (read == -1) { + break; + } + total += read; + arrowBuf.setBytes(!arrowBuf.hasArray()? 0: arrowBuf.arrayOffset(), new ByteArrayInputStream(bytes, 0, read), read); } + holder.end = total; + holder.buffer = arrowBuf; + varcharVector.set(rowCount, holder); + varcharVector.setIndexDefined(rowCount); } else { varcharVector.setNull(rowCount); } } - private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException { - varBinaryVector.setValueCount(rowCount + 1); - if (isNonNull && blob != null) { - byte[] data = blob.getBytes(0, (int) blob.length()); - varBinaryVector.setIndexDefined(rowCount); - varBinaryVector.setValueLengthSafe(rowCount, (int) blob.length()); - varBinaryVector.setSafe(rowCount, data); - } else { - varBinaryVector.setNull(rowCount); - } + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException, IOException { + updateVector(varBinaryVector, blob.getBinaryStream(), isNonNull, rowCount); } } \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java index a36be61c55a..e5b752fbc48 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java @@ -166,84 +166,84 @@ public void testDBValues() { } } - public void sqlToArrowTestInt() throws SQLException { + public void sqlToArrowTestInt() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1 = 101", true, "INT_FIELD1"), new RootAllocator(Integer.MAX_VALUE))){ assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); } } - public void sqlToArrowTestBool() throws SQLException { + public void sqlToArrowTestBool() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2 = 1", true, "BOOL_FIELD2"), new RootAllocator(Integer.MAX_VALUE))){ assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); } } - public void sqlToArrowTestTinyInts() throws SQLException { + public void sqlToArrowTestTinyInts() throws SQLException, IOException { try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3 = 45", true, "TINYINT_FIELD3"), new RootAllocator(Integer.MAX_VALUE))){ assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); } } - public void sqlToArrowTestSmallInts() throws SQLException { + public void sqlToArrowTestSmallInts() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4 = 12000", true, "SMALLINT_FIELD4"), new RootAllocator(Integer.MAX_VALUE))){ assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); } } - public void sqlToArrowTestBigInts() throws SQLException { + public void sqlToArrowTestBigInts() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIGINT_FIELD5"), new RootAllocator(Integer.MAX_VALUE))){ assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); } } - public void sqlToArrowTestBigDecimals() throws SQLException { + public void sqlToArrowTestBigDecimals() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DECIMAL_FIELD6"), new RootAllocator(Integer.MAX_VALUE))){ assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); } } - public void sqlToArrowTestDoubles() throws SQLException { + public void sqlToArrowTestDoubles() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DOUBLE_FIELD7"), new RootAllocator(Integer.MAX_VALUE))){ assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); } } - public void sqlToArrowTestRealValues() throws SQLException { + public void sqlToArrowTestRealValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "REAL_FIELD8"), new RootAllocator(Integer.MAX_VALUE))){ assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); } } - public void sqlToArrowTestTimeValues() throws SQLException { + public void sqlToArrowTestTimeValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIME_FIELD9"), new RootAllocator(Integer.MAX_VALUE))){ assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); } } - public void sqlToArrowTestDateValues() throws SQLException { + public void sqlToArrowTestDateValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DATE_FIELD10"), new RootAllocator(Integer.MAX_VALUE))){ assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); } } - public void sqlToArrowTestTimestampValues() throws SQLException { + public void sqlToArrowTestTimestampValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIMESTAMP_FIELD11"), new RootAllocator(Integer.MAX_VALUE))){ assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); } } - public void sqlToArrowTestByteValues() throws SQLException { + public void sqlToArrowTestByteValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BINARY_FIELD12"), new RootAllocator(Integer.MAX_VALUE))){ assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); @@ -251,42 +251,42 @@ public void sqlToArrowTestByteValues() throws SQLException { } } - public void sqlToArrowTestVarCharValues() throws SQLException { + public void sqlToArrowTestVarCharValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "VARCHAR_FIELD13"), new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); } } - public void sqlToArrowTestBlobValues() throws SQLException { + public void sqlToArrowTestBlobValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BLOB_FIELD14"), new RootAllocator(Integer.MAX_VALUE))){ assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); } } - public void sqlToArrowTestClobValues() throws SQLException { + public void sqlToArrowTestClobValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CLOB_FIELD15"), new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); } } - public void sqlToArrowTestCharValues() throws SQLException { + public void sqlToArrowTestCharValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CHAR_FIELD16"), new RootAllocator(Integer.MAX_VALUE))){ assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); } } - public void sqlToArrowTestBits() throws SQLException { + public void sqlToArrowTestBits() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIT_FIELD17"), new RootAllocator(Integer.MAX_VALUE))){ assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); } } - public void sqlToArrowTestNullValues() throws SQLException { + public void sqlToArrowTestNullValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery (false, table.getAllColumns()), new RootAllocator(Integer.MAX_VALUE))){ assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); @@ -309,7 +309,7 @@ public void sqlToArrowTestNullValues() throws SQLException { } } - public void sqlToArrowTestValuesWithPSTTimeZone() throws SQLException { + public void sqlToArrowTestValuesWithPSTTimeZone() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("PST")))){ @@ -320,7 +320,7 @@ public void sqlToArrowTestValuesWithPSTTimeZone() throws SQLException { } } - public void sqlToArrowTestValuesWithESTTimeZone() throws SQLException { + public void sqlToArrowTestValuesWithESTTimeZone() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("EST")))){ @@ -331,7 +331,7 @@ public void sqlToArrowTestValuesWithESTTimeZone() throws SQLException { } } - public void sqlToArrowTestValuesWithGMTTimeZone() throws SQLException { + public void sqlToArrowTestValuesWithGMTTimeZone() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("GMT")))){ @@ -342,7 +342,7 @@ public void sqlToArrowTestValuesWithGMTTimeZone() throws SQLException { } } - public void sqlToArrowTestSelectedColumnsNullValues() throws SQLException { + public void sqlToArrowTestSelectedColumnsNullValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("INT_FIELD1 = 102", true, table.getAllColumns()), new RootAllocator(Integer.MAX_VALUE))){ assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); @@ -405,12 +405,29 @@ private String getQuery (String whereClause, boolean isNotNull, String... column private String getQuery(String whereClause, boolean isNotNull, int index, String... columns) { StringBuilder query = new StringBuilder(String.format("select %s from %s ", - columns.length > 1 ? String.join(",", columns) : columns[index], table.getName())); +// columns.length > 1 ? String.join(",", columns) : columns[index], table.getName())); + columns.length > 1 ? join(",", columns) : columns[index], table.getName())); query.append(whereClause != null && !whereClause.isEmpty() ? " where " + whereClause + " and " : " where "); columns = columns.length == 1 && columns[0].contains(",") ? columns[0].split(",") : columns; query.append(isNotNull ? columns[index] + " is not null;" : columns[index] + " is null;"); return query.toString(); - } + } + + private static String join(String delimeter, String... columns) { + if (columns.length == 1) { + return columns[0]; + } + StringBuilder builder = new StringBuilder(); + int i = 0; + for (String column: columns) { + if (i > 0) { + builder.append(delimeter).append(" "); + } + builder.append(column); + i++; + } + return builder.toString(); + } } From acda6e383cc8100ec4935c94fb35fdefa54cf0d0 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 25 Apr 2018 15:26:22 -0700 Subject: [PATCH 099/129] Fixed a bug related to ArrowBuf offset. --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index d6adc3c9223..7e3d009e860 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -47,7 +47,6 @@ import org.apache.arrow.vector.holders.NullableSmallIntHolder; import org.apache.arrow.vector.holders.NullableTimeMilliHolder; import org.apache.arrow.vector.holders.NullableTinyIntHolder; -import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.holders.VarBinaryHolder; import org.apache.arrow.vector.holders.VarCharHolder; @@ -92,6 +91,7 @@ public class JdbcToArrowUtils { private static final int DEFAULT_BUFFER_SIZE = 256; private static final int DEFAULT_STREAM_BUFFER_SIZE = 1024; + private static final int DEFAULT_CLOB_SUBSTRING_READ_SIZE = 256; /** * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. @@ -462,17 +462,6 @@ private static void updateVector(TimeStampVector timeStampVector, Timestamp time } } - private static void updateVector(VarBinaryVector varBinaryVector, byte[] bytes, boolean isNonNull, int rowCount) { - varBinaryVector.setValueCount(rowCount + 1); - if (isNonNull && bytes != null) { - varBinaryVector.setIndexDefined(rowCount); - varBinaryVector.setValueLengthSafe(rowCount, bytes.length); - varBinaryVector.setSafe(rowCount, bytes); - } else { - varBinaryVector.setNull(rowCount); - } - } - private static void updateVector(VarBinaryVector varBinaryVector, InputStream is, boolean isNonNull, int rowCount) throws IOException { varBinaryVector.setValueCount(rowCount + 1); if (isNonNull && is != null) { @@ -486,8 +475,8 @@ private static void updateVector(VarBinaryVector varBinaryVector, InputStream is if (read == -1) { break; } + arrowBuf.setBytes(total, new ByteArrayInputStream(bytes, 0, read), read); total += read; - arrowBuf.setBytes(!arrowBuf.hasArray()? 0: arrowBuf.arrayOffset(), new ByteArrayInputStream(bytes, 0, read), read); } holder.end = total; holder.buffer = arrowBuf; @@ -503,19 +492,19 @@ private static void updateVector(VarCharVector varcharVector, Clob clob, boolean if (isNonNull && clob != null) { VarCharHolder holder = new VarCharHolder(); ArrowBuf arrowBuf = varcharVector.getDataBuffer(); - InputStream is = clob.getAsciiStream(); holder.start = 0; - byte[] bytes = new byte[DEFAULT_STREAM_BUFFER_SIZE]; - int total = 0; - while (true) { - int read = is.read(bytes, 0, DEFAULT_STREAM_BUFFER_SIZE); - if (read == -1) { - break; - } - total += read; - arrowBuf.setBytes(!arrowBuf.hasArray()? 0: arrowBuf.arrayOffset(), new ByteArrayInputStream(bytes, 0, read), read); + long length = clob.length(); + int read = 1; + int readSize = length < DEFAULT_CLOB_SUBSTRING_READ_SIZE? (int)length: DEFAULT_CLOB_SUBSTRING_READ_SIZE; + int totalBytes = 0; + while (read <= length) { + String str = clob.getSubString(read, readSize); + byte[] bytes = str.getBytes(StandardCharsets.UTF_8); + arrowBuf.setBytes(totalBytes, new ByteArrayInputStream(bytes, 0, bytes.length), bytes.length); + totalBytes += bytes.length; + read += readSize; } - holder.end = total; + holder.end = totalBytes; holder.buffer = arrowBuf; varcharVector.set(rowCount, holder); varcharVector.setIndexDefined(rowCount); From f06d85e25256a2f524bf6a830dc8f3d894ccbd04 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Wed, 25 Apr 2018 15:45:05 -0700 Subject: [PATCH 100/129] Removed unused import. --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 7e3d009e860..4608d7261d6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -59,7 +59,6 @@ import org.apache.arrow.vector.util.DecimalUtility; import java.io.ByteArrayInputStream; -import java.io.IOError; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; From 4bfbfe6bc8a847fb9fc57bb3d0471ca75d01baea Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:16:14 -0400 Subject: [PATCH 101/129] file committed for review comment implementation --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 4608d7261d6..a96e45e6bd0 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -513,7 +513,7 @@ private static void updateVector(VarCharVector varcharVector, Clob clob, boolean } private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException, IOException { - updateVector(varBinaryVector, blob.getBinaryStream(), isNonNull, rowCount); + updateVector(varBinaryVector, blob != null ? blob.getBinaryStream() : null, isNonNull, rowCount); } -} \ No newline at end of file +} From 4329c4bb7432b6a0a9be2b049f6713193c1ebd3d Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:23:35 -0400 Subject: [PATCH 102/129] file committed for review comment implementation --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 64e153bf562..8eaaa703da2 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -21,7 +21,6 @@ import java.math.BigDecimal; import java.util.Arrays; -import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -45,43 +44,51 @@ */ public class JdbcToArrowTestHelper { - public static void assertIntVectorValues(IntVector intVector, int rowCount, int [] values) { + public static void assertIntVectorValues(IntVector intVector, int rowCount, Integer[] values) { assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - assertEquals(values[j], intVector.get(j)); + assertEquals(values[j].intValue(), intVector.get(j)); } } - public static void assertBitBooleanVectorValues(BitVector bitVector, int rowCount, int[] values){ + public static void assertBooleanVectorValues(BitVector bitVector, int rowCount, Boolean[] values){ assertEquals(rowCount, bitVector.getValueCount()); for(int j = 0; j < bitVector.getValueCount(); j++){ - assertEquals(values[j], bitVector.get(j)); + assertEquals(values[j].booleanValue(), bitVector.get(j) == 1); } } - public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, int[] values){ + public static void assertBitVectorValues(BitVector bitVector, int rowCount, Integer[] values){ + assertEquals(rowCount, bitVector.getValueCount()); + + for(int j = 0; j < bitVector.getValueCount(); j++){ + assertEquals(values[j].intValue(), bitVector.get(j)); + } + } + + public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, Integer[] values){ assertEquals(rowCount, tinyIntVector.getValueCount()); for(int j = 0; j < tinyIntVector.getValueCount(); j++){ - assertEquals(values[j], tinyIntVector.get(j)); + assertEquals(values[j].intValue(), tinyIntVector.get(j)); } } - public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, int[] values){ + public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, Integer[] values){ assertEquals(rowCount, smallIntVector.getValueCount()); for(int j = 0; j < smallIntVector.getValueCount(); j++){ - assertEquals(values[j], smallIntVector.get(j)); + assertEquals(values[j].intValue(), smallIntVector.get(j)); } } - public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, int[] values){ + public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, Long[] values){ assertEquals(rowCount, bigIntVector.getValueCount()); for(int j = 0; j < bigIntVector.getValueCount(); j++){ - assertEquals(values[j], bigIntVector.get(j)); + assertEquals(values[j].longValue(), bigIntVector.get(j)); } } @@ -94,7 +101,7 @@ public static void assertDecimalVectorValues(DecimalVector decimalVector, int ro } } - public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, double[] values){ + public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, Double[] values){ assertEquals(rowCount, float8Vector.getValueCount()); for(int j = 0; j < float8Vector.getValueCount(); j++){ @@ -102,7 +109,7 @@ public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCo } } - public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, float[] values){ + public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, Float[] values){ assertEquals(rowCount, float4Vector.getValueCount()); for(int j = 0; j < float4Vector.getValueCount(); j++){ @@ -110,27 +117,27 @@ public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCo } } - public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, long[] values){ + public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, Long[] values){ assertEquals(rowCount, timeMilliVector.getValueCount()); for(int j = 0; j < timeMilliVector.getValueCount(); j++){ - assertEquals(values[j], timeMilliVector.get(j)); + assertEquals(values[j].longValue(), timeMilliVector.get(j)); } } - public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, long[] values){ + public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, Long[] values){ assertEquals(rowCount, dateMilliVector.getValueCount()); for(int j = 0; j < dateMilliVector.getValueCount(); j++){ - assertEquals(values[j], dateMilliVector.get(j)); + assertEquals(values[j].longValue(), dateMilliVector.get(j)); } } - public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, long[] values){ + public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, Long[] values){ assertEquals(rowCount, timeStampVector.getValueCount()); for(int j = 0; j < timeStampVector.getValueCount(); j++){ - assertEquals(values[j], timeStampVector.get(j)); + assertEquals(values[j].longValue(), timeStampVector.get(j)); } } From 840970cab8125e5b74174c0523a3849f75f7ab2c Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:23:54 -0400 Subject: [PATCH 103/129] file committed for review comment implementation --- .../org/apache/arrow/adapter/jdbc/Table.java | 357 ++++++++---------- 1 file changed, 153 insertions(+), 204 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 76940ba04a9..3c99ceb1578 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,254 +30,203 @@ public class Table { private String name; + private String type; + private String vector; + private String timezone; private String create; private String[] data; private String query; private String drop; - private int [] ints; - private int [] booleans; - private int [] tinyInts; - private int [] smallInts; - private int [] bigInts; - private long [] times; - private long [] dates; - private long [] timestamps; - private String [] bytes; - private String [] varchars; - private String [] chars; - private String [] clobs; - private float [] reals; - private double [] doubles; - private BigDecimal [] decimals; - private String allColumns; - private long [] pstTime; - private long [] estTime; - private long [] gmtTime; - private long [] pstDate; - private long [] estDate; - private long [] gmtDate; - private long [] pstTimestamp; - private long [] estTimestamp; - private long [] gmtTimestamp; - - public Table() { + private String[] values; + private String[] vectors; + private int rowCount; + + public Table() { } - public String getName() { + public String getName() { return name; } + public void setName(String name) { this.name = name; } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getVector() { + return vector; + } + + public void setVector(String vector) { + this.vector = vector; + } + + public String[] getValues() { + return values; + } + + public void setValues(String[] values) { + this.values = values; + } + + public Long[] getLongValues() { + Long[] arr = new Long[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = Long.parseLong(str); + } + return arr; + } + + public Integer[] getIntValues() { + Integer[] arr = new Integer[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = Integer.parseInt(str); + } + return arr; + } + public Boolean[] getBoolValues() { + Boolean[] arr = new Boolean[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = Boolean.parseBoolean(str); + } + return arr; + } + public BigDecimal[] getBigDecimalValues() { + BigDecimal[] arr = new BigDecimal[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = new BigDecimal(str); + } + return arr; + } + public Double[] getDoubleValues() { + Double[] arr = new Double[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = Double.parseDouble(str); + } + return arr; + } + + public Float[] getFloatValues() { + Float[] arr = new Float[values.length]; + int i = 0; + for (String str: values) { + arr[i++] = Float.parseFloat(str); + } + return arr; + } + + public byte[][] getBinaryValues() { + return getHexToByteArray(values); + } + + public byte[][] getVarCharValues() { + return getByteArray(values); + } + + public byte[][] getBlobValues() { + return getBinaryValues(); + } + + public byte[][] getClobValues() { + return getByteArray(values); + } + + public byte[][] getCharValues() { + return getByteArray(values); + } + public String getCreate() { return create; } + public void setCreate(String create) { this.create = create; } + public String[] getData() { return data; } + public void setData(String[] data) { this.data = data; } + public String getQuery() { return query; } + public void setQuery(String query) { this.query = query; } + public String getDrop() { return drop; } + public void setDrop(String drop) { this.drop = drop; } - public int[] getInts() { - return ints; - } - public void setInts(int[] ints) { - this.ints = ints; - } - public int[] getBooleans() { - return booleans; - } - public void setBooleans(int[] booleans) { - this.booleans = booleans; - } - public BigDecimal [] getDecimals() { - return decimals; - } - public void setDecimals(BigDecimal [] decimals) { - this.decimals = decimals; - } - public double[] getDoubles() { - return doubles; - } - public void setDoubles(double[] doubles) { - this.doubles = doubles; - } - public int[] getTinyInts() { - return tinyInts; - } - public void setTinyInts(int[] tinyInts) { - this.tinyInts = tinyInts; - } - public int[] getSmallInts() { - return smallInts; - } - public void setSmallInts(int[] smallInts) { - this.smallInts = smallInts; - } - public int[] getBigInts() { - return bigInts; - } - public void setBigInts(int[] bigInts) { - this.bigInts = bigInts; - } - public long[] getTimes() { - return times; - } - public void setTimes(long[] times) { - this.times = times; - } - public long[] getDates() { - return dates; - } - public void setDates(long[] dates) { - this.dates = dates; - } - public long[] getTimestamps() { - return timestamps; - } - public void setTimestamps(long[] timestamps) { - this.timestamps = timestamps; - } - public String [] getBytes() { - return bytes; - } - public void setBytes(String [] bytes) { - this.bytes = bytes; - } - public String []getVarchars() { - return varchars; - } - public void setVarchars(String [] varchars) { - this.varchars = varchars; - } - public String [] getChars() { - return chars; - } - public void setChars(String [] chars) { - this.chars = chars; - } - public String [] getClobs() { - return clobs; - } - public void setClobs(String [] clobs) { - this.clobs = clobs; - } - public float[] getReals() { - return reals; - } - public void setReals(float[] reals) { - this.reals = reals; - } - public String getAllColumns() { - return allColumns; - } - public void setAllColumns(String allColumns) { - this.allColumns = allColumns; - } - public byte [][] getHexStringAsByte () { - return getHexToByteArray (bytes); - } - public byte [][] getClobAsByte () { - return getByteArray (clobs); - } - public byte [][] getCharAsByte () { - return getByteArray (chars); - } - public byte [][] getVarCharAsByte () { - return getByteArray (varchars); - } - public long[] getPstTime() { - return pstTime; - } - public void setPstTime(long[] pstTime) { - this.pstTime = pstTime; - } - public long[] getEstTime() { - return estTime; - } - public void setEstTime(long[] estTime) { - this.estTime = estTime; - } - public long[] getGmtTime() { - return gmtTime; - } - public void setGmtTime(long[] gmtTime) { - this.gmtTime = gmtTime; - } - public long[] getPstDate() { - return pstDate; - } - public void setPstDate(long[] pstDate) { - this.pstDate = pstDate; - } - public long[] getEstDate() { - return estDate; - } - public void setEstDate(long[] estDate) { - this.estDate = estDate; - } - public long[] getGmtDate() { - return gmtDate; - } - public void setGmtDate(long[] gmtDate) { - this.gmtDate = gmtDate; - } - public long[] getPstTimestamp() { - return pstTimestamp; + + public String getTimezone() { + return timezone; } - public void setPstTimestamp(long[] pstTimestamp) { - this.pstTimestamp = pstTimestamp; + + public void setTimezone(String timezone) { + this.timezone = timezone; } - public long[] getEstTimestamp() { - return estTimestamp; + + public String[] getVectors() { + return vectors; } - public void setEstTimestamp(long[] estTimestamp) { - this.estTimestamp = estTimestamp; + + public void setVectors(String[] vectors) { + this.vectors = vectors; } - public long[] getGmtTimestamp() { - return gmtTimestamp; + + public int getRowCount() { + return rowCount; } - public void setGmtTimestamp(long[] gmtTimestamp) { - this.gmtTimestamp = gmtTimestamp; + + public void setRowCount(int rowCount) { + this.rowCount = rowCount; } - - private byte [][] getByteArray (String [] data) { - byte [][] byteArr = new byte [data.length][]; - - for (int i = 0; i < data.length; i++) { - byteArr [i] = data [i].getBytes(StandardCharsets.UTF_8); - } - return byteArr; + + private byte[][] getByteArray(String[] data) { + byte[][] byteArr = new byte[data.length][]; + + for (int i = 0; i < data.length; i++) { + byteArr[i] = data[i].getBytes(StandardCharsets.UTF_8); + } + return byteArr; } - private byte [][] getHexToByteArray (String [] data){ - byte [][] byteArr = new byte [data.length][]; - - for (int i = 0; i < data.length; i++) { - byteArr [i] = hexStringToByteArray(data [i]); - } - return byteArr; - } - private static byte[] hexStringToByteArray(String s) { + + private byte[][] getHexToByteArray(String[] data) { + byte[][] byteArr = new byte[data.length][]; + + for (int i = 0; i < data.length; i++) { + byteArr[i] = hexStringToByteArray(data[i]); + } + return byteArr; + } + + private static byte[] hexStringToByteArray(String s) { int len = s.length(); byte[] data = new byte[len / 2]; for (int i = 0; i < len; i += 2) { data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) - + Character.digit(s.charAt(i+1), 16)); + + Character.digit(s.charAt(i + 1), 16)); } return data; } From 2069103d17d01d90ff9e25621bcd79ee4f0a1109 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:24:36 -0400 Subject: [PATCH 104/129] file committed for review comment implementation --- .../jdbc/h2/JdbcToArrowDataTypesTest.java | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java new file mode 100644 index 00000000000..6d7feeef97b --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBooleanVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; + +/** + * + * JUnit Test class to test various datatypes converted into Arrow vector for H2 database + * + */ +@RunWith(Parameterized.class) +public class JdbcToArrowDataTypesTest { + private Connection conn = null; + private Table table; + + private static final String BIGINT = "big_int"; + private static final String BINARY = "binary"; + private static final String BIT = "bit"; + private static final String BLOB = "blob"; + private static final String BOOL = "bool"; + private static final String CHAR = "char"; + private static final String CLOB = "clob"; + private static final String DATE = "date"; + private static final String DECIMAL = "decimal"; + private static final String DOUBLE = "double"; + private static final String INT = "int"; + private static final String REAL = "real"; + private static final String SMALLINT = "small_int"; + private static final String TIME = "time"; + private static final String TIMESTAMP = "timestamp"; + private static final String TINYINT = "tiny_int"; + private static final String VARCHAR = "varchar"; + + private static final String[] testFiles = { + "h2/test1_bigint_h2.yml", + "h2/test1_binary_h2.yml", + "h2/test1_bit_h2.yml", + "h2/test1_blob_h2.yml", + "h2/test1_bool_h2.yml", + "h2/test1_char_h2.yml", + "h2/test1_clob_h2.yml", + "h2/test1_date_h2.yml", + "h2/test1_decimal_h2.yml", + "h2/test1_double_h2.yml", + "h2/test1_int_h2.yml", + "h2/test1_real_h2.yml", + "h2/test1_smallint_h2.yml", + "h2/test1_time_h2.yml", + "h2/test1_timestamp_h2.yml", + "h2/test1_tinyint_h2.yml", + "h2/test1_varchar_h2.yml" + }; + + /** + * Constructor which populate table object for each test iteration + * @param table + */ + public JdbcToArrowDataTypesTest(Table table) { + this.table = table; + } + + /** + * This method creates Table object after reading YAML file + * @param ymlFilePath + * @return + * @throws IOException + */ + private static Table getTable(String ymlFilePath) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + } + + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + /** + * Clean up method to close connection after test completes + * @throws SQLException + */ + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile: testFiles) { + tableArr[i++] = new Object[]{getTable(testFile)}; + } + return Arrays.asList(tableArr); + } + + /** + * This method tests various datatypes converted into Arrow vector for H2 database + * @throws SQLException + * @throws IOException + */ + @Test + public void testDBValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + switch (table.getType()) { + case BIGINT: + assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case BINARY:case BLOB: + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, table.getBinaryValues()); + break; + case BIT: + assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case BOOL: + assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getBoolValues()); + break; + case CHAR:case VARCHAR: case CLOB: + assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, table.getCharValues()); + break; + case DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case DECIMAL: + assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, table.getBigDecimalValues()); + break; + case DOUBLE: + assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, table.getDoubleValues()); + break; + case INT: + assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case SMALLINT: + assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case TINYINT: + assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case REAL: + assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, table.getFloatValues()); + break; + } + } + } + +} + From 95f8e236a573e298cb8aeeb2c6280914181447f4 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:25:41 -0400 Subject: [PATCH 105/129] file committed for review comment implementation --- .../h2/test1_all_datatypes_null_h2.yml | 28 +++++- .../src/test/resources/h2/test1_bigint_h2.yml | 77 ++++++--------- .../src/test/resources/h2/test1_binary_h2.yml | 79 ++++++--------- .../src/test/resources/h2/test1_bit_h2.yml | 79 ++++++--------- .../src/test/resources/h2/test1_blob_h2.yml | 77 ++++++--------- .../src/test/resources/h2/test1_bool_h2.yml | 77 ++++++--------- .../src/test/resources/h2/test1_char_h2.yml | 79 ++++++--------- .../src/test/resources/h2/test1_clob_h2.yml | 77 ++++++--------- .../src/test/resources/h2/test1_date_h2.yml | 79 ++++++--------- .../test/resources/h2/test1_decimal_h2.yml | 79 ++++++--------- .../src/test/resources/h2/test1_double_h2.yml | 79 ++++++--------- .../test/resources/h2/test1_est_date_h2.yml | 48 ++++++++++ .../test/resources/h2/test1_est_time_h2.yml | 48 ++++++++++ .../resources/h2/test1_est_timestamp_h2.yml | 49 ++++++++++ .../test/resources/h2/test1_gmt_date_h2.yml | 48 ++++++++++ .../test/resources/h2/test1_gmt_time_h2.yml | 48 ++++++++++ .../resources/h2/test1_gmt_timestamp_h2.yml | 48 ++++++++++ .../src/test/resources/h2/test1_int_h2.yml | 70 ++++---------- .../test/resources/h2/test1_pst_date_h2.yml | 48 ++++++++++ .../test/resources/h2/test1_pst_time_h2.yml | 48 ++++++++++ .../resources/h2/test1_pst_timestamp_h2.yml | 48 ++++++++++ .../src/test/resources/h2/test1_real_h2.yml | 79 ++++++--------- .../h2/test1_selected_datatypes_null_h2.yml | 46 +++++++++ .../test/resources/h2/test1_smallint_h2.yml | 77 ++++++--------- .../src/test/resources/h2/test1_time_h2.yml | 79 ++++++--------- .../test/resources/h2/test1_timestamp_h2.yml | 79 ++++++--------- .../test/resources/h2/test1_tinyint_h2.yml | 77 ++++++--------- .../test/resources/h2/test1_varchar_h2.yml | 79 ++++++--------- .../src/test/resources/h2/testcases_h2.txt | 96 +++++++++++++++++++ 29 files changed, 1072 insertions(+), 853 deletions(-) create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_est_time_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_est_timestamp_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_gmt_time_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_gmt_timestamp_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_pst_time_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_pst_timestamp_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_selected_datatypes_null_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_null_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_null_h2.yml index ea4dce2a87c..977879df491 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_null_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_null_h2.yml @@ -11,6 +11,29 @@ name: 'table1' +type: 'null' + +vectors: + - 'INT_FIELD1' + - 'BOOL_FIELD2' + - 'TINYINT_FIELD3' + - 'SMALLINT_FIELD4' + - 'BIGINT_FIELD5' + - 'DECIMAL_FIELD6' + - 'DOUBLE_FIELD7' + - 'REAL_FIELD8' + - 'TIME_FIELD9' + - 'DATE_FIELD10' + - 'TIMESTAMP_FIELD11' + - 'BINARY_FIELD12' + - 'VARCHAR_FIELD13' + - 'BLOB_FIELD14' + - 'CLOB_FIELD15' + - 'CHAR_FIELD16' + - 'BIT_FIELD17' + +rowCount: '5' + create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' @@ -21,11 +44,6 @@ data: - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - 'INSERT INTO table1 VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' - - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml index 88e3053a7d2..066bececfd7 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bigint_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'big_int' + +vector: 'BIGINT_FIELD5' + +create: 'CREATE TABLE table1 (bigint_field5 BIGINT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' + - 'INSERT INTO table1 VALUES (92233720);' query: 'select bigint_field5 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' + - '92233720' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml index d360773c55f..ce3e4f12717 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_binary_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'binary' + +vector: 'BINARY_FIELD12' + +create: 'CREATE TABLE table1 (binary_field12 BINARY(100));' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + +query: 'select binary_field12 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml index d360773c55f..aeb7a20e0b3 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bit_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'bit' + +vector: 'BIT_FIELD17' + +create: 'CREATE TABLE table1 (bit_field17 BIT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + +query: 'select bit_field17 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' + - '1' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml index 0416a2e7fd6..b4cd2ca80ca 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_blob_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'blob' + +vector: 'BLOB_FIELD14' + +create: 'CREATE TABLE table1 (blob_field14 BLOB);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' + - 'INSERT INTO table1 VALUES (''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'');' query: 'select blob_field14 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml index ea0956996b4..8219a55ecab 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_bool_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'bool' + +vector: 'BOOL_FIELD2' + +create: 'CREATE TABLE table1 (bool_field2 BOOLEAN);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' + - 'INSERT INTO table1 VALUES (1);' query: 'select bool_field2 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' + - 'true' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml index d360773c55f..6e2cb185c3b 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_char_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'char' + +vector: 'CHAR_FIELD16' + +create: 'CREATE TABLE table1 (char_field16 CHAR(16));' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + - 'INSERT INTO table1 VALUES (''some char text'');' + +query: 'select char_field16 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' + - 'some char text' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml index 1518e8c1ed5..57c69ffe06d 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_clob_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'clob' + +vector: 'CLOB_FIELD15' + +create: 'CREATE TABLE table1 (clob_field15 CLOB);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to clob'');' query: 'select CLOB_FIELD15 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' + - 'some text that needs to be converted to clob' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml index d360773c55f..45aa56c7417 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'date' + +vector: 'DATE_FIELD10' + +create: 'CREATE TABLE table1 (date_field10 DATE);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + +query: 'select date_field10 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml index d360773c55f..3ee15c40942 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_decimal_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'decimal' + +vector: 'DECIMAL_FIELD6' + +create: 'CREATE TABLE table1 (decimal_field6 DECIMAL(20,2));' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + - 'INSERT INTO table1 VALUES (17345667789.23);' + +query: 'select decimal_field6 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' + - '17345667789.23' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml index d360773c55f..f4190092871 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_double_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'double' + +vector: 'DOUBLE_FIELD7' + +create: 'CREATE TABLE table1 (double_field7 DOUBLE);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + +query: 'select double_field7 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' + - '56478356785.345' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml new file mode 100644 index 00000000000..290c32ebafa --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'est_date' + +timezone: 'EST' + +vector: 'DATE_FIELD10' + +create: 'CREATE TABLE table1 (date_field10 DATE);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + +query: 'select date_field10 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' + - '1518411600000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_est_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_est_time_h2.yml new file mode 100644 index 00000000000..c6fc7a18035 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_est_time_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'est_time' + +timezone: 'EST' + +vector: 'TIME_FIELD9' + +create: 'CREATE TABLE table1 (time_field9 TIME);' + +data: + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + +query: 'select time_field9 from table1;' + +drop: 'DROP table table1;' + +values: + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' + - '63935000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_est_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_est_timestamp_h2.yml new file mode 100644 index 00000000000..b0ec5b70839 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_est_timestamp_h2.yml @@ -0,0 +1,49 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'est_timestamp' + +timezone: 'EST' + +vector: 'TIMESTAMP_FIELD11' + +create: 'CREATE TABLE table1 (timestamp_field11 TIMESTAMP);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + +query: 'select timestamp_field11 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + - '1518457535000' + \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml new file mode 100644 index 00000000000..03929c936e4 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'gmt_date' + +timezone: 'GMT' + +vector: 'DATE_FIELD10' + +create: 'CREATE TABLE table1 (date_field10 DATE);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + +query: 'select date_field10 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' + - '1518393600000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_gmt_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_time_h2.yml new file mode 100644 index 00000000000..ae28c51e285 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_time_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'gmt_time' + +timezone: 'GMT' + +vector: 'TIME_FIELD9' + +create: 'CREATE TABLE table1 (time_field9 TIME);' + +data: + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + +query: 'select time_field9 from table1;' + +drop: 'DROP table table1;' + +values: + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_gmt_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_timestamp_h2.yml new file mode 100644 index 00000000000..b468f5af948 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_timestamp_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'gmt_timestamp' + +timezone: 'GMT' + +vector: 'TIMESTAMP_FIELD11' + +create: 'CREATE TABLE table1 (timestamp_field11 TIMESTAMP);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + +query: 'select timestamp_field11 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml index b8d06492ac1..7fb6d686720 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_int_h2.yml @@ -11,58 +11,25 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'int' + +vector: 'INT_FIELD1' + +create: 'CREATE TABLE table1 (int_field1 INT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' + - 'INSERT INTO table1 VALUES (101);' -integers: +values: - '101' - '101' - '101' @@ -73,11 +40,6 @@ integers: - '101' - '101' - '101' - - '101' - - '101' - - '101' - - '101' - - '101' query: 'select int_field1 from table1;' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml new file mode 100644 index 00000000000..81a668f37a4 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'pst_date' + +timezone: 'PST' + +vector: 'DATE_FIELD10' + +create: 'CREATE TABLE table1 (date_field10 DATE);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + - 'INSERT INTO table1 VALUES (''2018-02-12'');' + +query: 'select date_field10 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' + - '1518422400000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_pst_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_pst_time_h2.yml new file mode 100644 index 00000000000..058d54d2027 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_pst_time_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'pst_time' + +timezone: 'PST' + +vector: 'TIME_FIELD9' + +create: 'CREATE TABLE table1 (time_field9 TIME);' + +data: + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + +query: 'select time_field9 from table1;' + +drop: 'DROP table table1;' + +values: + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' + - '74735000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_pst_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_pst_timestamp_h2.yml new file mode 100644 index 00000000000..19b6b5f4405 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_pst_timestamp_h2.yml @@ -0,0 +1,48 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'pst_timestamp' + +timezone: 'PST' + +vector: 'TIMESTAMP_FIELD11' + +create: 'CREATE TABLE table1 (timestamp_field11 TIMESTAMP);' + +data: + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + +query: 'select timestamp_field11 from table1;' + +drop: 'DROP table table1;' + +values: + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' + - '1518468335000' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml index d360773c55f..c8f8aeb7810 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_real_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'real' + +vector: 'REAL_FIELD8' + +create: 'CREATE TABLE table1 (real_field8 REAL);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + - 'INSERT INTO table1 VALUES (56478356785.345);' + +query: 'select real_field8 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' + - '56478356785.345f' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_selected_datatypes_null_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_selected_datatypes_null_h2.yml new file mode 100644 index 00000000000..93b1aae556f --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_selected_datatypes_null_h2.yml @@ -0,0 +1,46 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'selected_null_column' + +vectors: + - 'BIGINT_FIELD5' + - 'DECIMAL_FIELD6' + - 'DOUBLE_FIELD7' + - 'REAL_FIELD8' + - 'TIME_FIELD9' + - 'DATE_FIELD10' + - 'TIMESTAMP_FIELD11' + - 'BINARY_FIELD12' + - 'VARCHAR_FIELD13' + - 'BLOB_FIELD14' + - 'CLOB_FIELD15' + - 'CHAR_FIELD16' + - 'BIT_FIELD17' + +rowCount: '5' + +create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, + decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, + binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' + +data: + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + - 'INSERT INTO table1 (int_field1, bool_field2, tinyint_field3, smallint_field4) VALUES (102, 0, 46, 12001);' + +query: 'select bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' + +drop: 'DROP table table1;' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml index 5fde24c5761..887c74f4db5 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_smallint_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'small_int' + +vector: 'SMALLINT_FIELD4' + +create: 'CREATE TABLE table1 (smallint_field4 SMALLINT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' + - 'INSERT INTO table1 VALUES (12000);' query: 'select smallint_field4 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' + - '12000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml index d360773c55f..c9baaee8dd1 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_time_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'time' + +vector: 'TIME_FIELD9' + +create: 'CREATE TABLE table1 (time_field9 TIME);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + - 'INSERT INTO table1 VALUES (''12:45:35'');' + +query: 'select time_field9 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' + - '45935000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml index d360773c55f..7d93faad1fa 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_timestamp_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'timestamp' + +vector: 'TIMESTAMP_FIELD11' + +create: 'CREATE TABLE table1 (timestamp_field11 TIMESTAMP);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + - 'INSERT INTO table1 VALUES (''2018-02-12 12:45:35'');' + +query: 'select timestamp_field11 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' + - '1518439535000' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml index 92c89967348..a419416c8f1 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_tinyint_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'tinyint' + +vector: 'TINYINT_FIELD3' + +create: 'CREATE TABLE table1 (tinyint_field3 TINYINT);' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' + - 'INSERT INTO table1 VALUES (45);' query: 'select tinyint_field3 from table1;' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +values: + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' + - '45' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml index d360773c55f..0bd142178cc 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_varchar_h2.yml @@ -11,57 +11,36 @@ name: 'table1' -create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, - decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, - binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT);' +type: 'varchar' + +vector: 'VARCHAR_FIELD13' + +create: 'CREATE TABLE table1 (varchar_field13 VARCHAR(256));' data: - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, ''12:45:35'', ''2018-02-12'', - ''2018-02-12 12:45:35'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + - 'INSERT INTO table1 VALUES (''some text that needs to be converted to varchar'');' + +query: 'select varchar_field13 from table1;' -query: 'select int_field1 from table1;' +drop: 'DROP table table1;' -drop: 'DROP table table1;' \ No newline at end of file +values: + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' + - 'some text that needs to be converted to varchar' diff --git a/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt b/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt new file mode 100644 index 00000000000..845b5ba717f --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt @@ -0,0 +1,96 @@ +#--- Created TABLE -1 : +CREATE TABLE H2Table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT, identity_field18 IDENTITY, varchar_ignorecase_field19 VARCHAR_IGNORECASE(30), UUID_field20 UUID, float_field21 FLOAT); + +#--- Inserted values in TABLE -1 : +INSERT INTO H2Table1 VALUES (101, 1, 41, 12001, 92233721, 17345667781.23, 56478356781.345, 56478356781.345, '1:45:35', '2018-02-1', '2018-02-1 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as varchar1', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as clob1', 'some char text1', 1, 10001, 'varchar_ignorecase_1', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); +INSERT INTO H2Table1 VALUES (102, 0, 42, 12002, 92233722, 17345667782.23, 56478356782.345, 56478356782.345, '2:45:35', '2018-02-2', '2018-02-2 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as varchar2', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as clob2', 'some char text2', 0, 10002, 'varchar_ignorecase_2', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); +INSERT INTO H2Table1 VALUES (103, 1, 43, 12003, 92233723, 17345667783.23, 56478356783.345, 56478356783.345, '3:45:35', '2018-02-3', '2018-02-3 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as varchar3', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as clob3', 'some char text3', 1, 10003, 'varchar_ignorecase_3', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13'); +INSERT INTO H2Table1 VALUES (104, 0, 44, 12004, 92233724, 17345667784.23, 56478356784.345, 56478356784.345, '4:45:35', '2018-02-4', '2018-02-4 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some as varchar4', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some text as clob4', 'some char text4', 0, 10004, 'varchar_ignorecase_4', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14'); +INSERT INTO H2Table1 VALUES (105, 1, 45, 12005, 92233725, 17345667785.23, 56478356785.345, 56478356785.345, '5:45:35', '2018-02-5', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as varchar5', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob5', 'some char text5', 1, 10005, 'varchar_ignorecase_5', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15'); + +#--- Created TABLE -2 : +CREATE TABLE H2Table2 (int_T2field1 INT, bool_T2field2 BOOLEAN, tinyint_T2field3 TINYINT, smallint_T2field4 SMALLINT, bigint_T2field5 BIGINT, decimal_T2field6 DECIMAL(20,2), double_T2field7 DOUBLE, real_T2field8 REAL, time_T2field9 TIME, date_T2field10 DATE, timestamp_T2field11 TIMESTAMP, binary_T2field12 BINARY(100), varchar_T2field13 VARCHAR(256), blob_T2field14 BLOB, clob_T2field15 CLOB, char_T2field16 CHAR(16), bit_T2field17 BIT, identity_T2field18 IDENTITY, varchar_ignorecase_T2field19 VARCHAR_IGNORECASE(30), UUID_T2field20 UUID, float_T2field21 FLOAT); + +#--- Inserted values in TABLE -2 : +INSERT INTO H2Table2 VALUES (106, 1, 46, 12006, 92233726, 17345667786.23, 56478356786.345, 56478356786.345, '6:45:35', '2018-02-6', '2018-02-6 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as varchar6', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as clob6', 'some char text6', 0, 10006, 'varchar_ignorecase_6', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16'); +INSERT INTO H2Table2 VALUES (107, 0, 47, 12007, 92233727, 17345667787.23, 56478356787.345, 56478356787.345, '7:45:35', '2018-02-7','2018-02-7 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as varchar7', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as clob7', 'some char text7', 1, 10007, 'varchar_ignorecase_7', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17'); +INSERT INTO H2Table2 VALUES (108, 1, 48, 02008, 92233728, 17345667788.23, 56478356788.345, 56478356788.345, '8:45:35', '2018-02-8', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as varchar8', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as clob8', 'some char text8', 1, 10008, 'varchar_ignorecase_8', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18'); +INSERT INTO H2Table2 VALUES (109, 0, 49, 12009, 92233729, 17345667789.23, 56478356789.345, 56478356789.345, '9:45:35', '2018-02-9', '2018-02-9 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as varchar9', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as clob9', 'some char text9', 0, 10009, 'varchar_ignorecase_9', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a19'); +INSERT INTO H2Table2 VALUES (110, 1, 50, 12010, 92233710, 17345667710.23, 56478356710.345, 56478356710.345, '10:45:35', '2018-02-10', '2018-02-10 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617210', 'some text as varchar10', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob10', 'some char text10', 1, 10010, 'varchar_ignorecase_10', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a10'); + +#--- Created TABLE -3 : +CREATE TABLE H2Table4 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); + +--- Inserted values in TABLE -3 : +INSERT INTO H2Table4 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); +INSERT INTO H2Table4 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); +INSERT INTO H2Table4 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); +INSERT INTO H2Table4 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); +INSERT INTO H2Table4 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); +INSERT INTO H2Table4 VALUES (214, 92233214, 17345667214.23, 56478356214.345, 'some text as varchar214'); + +#--- Created TABLE -4 : +CREATE TABLE H2Table5 (int_T5field1 INT, bigint_T5field2 BIGINT, decimal_T5field3 DECIMAL(20,2), float_T5field4 FLOAT, varchar_T5field5 VARCHAR(256)); + +#--- Inserted values in TABLE -4 : +INSERT INTO H2Table5 VALUES (206, 92233206, 17345667206.23, 56478356206.345, 'some text as varchar206'); +INSERT INTO H2Table5 VALUES (207, 92233207, 17345667207.23, 56478356207.345, 'some text as varchar207'); +INSERT INTO H2Table5 VALUES (208, 92233208, 17345667208.23, 56478356208.345, 'some text as varchar208'); +INSERT INTO H2Table5 VALUES (209, 92233209, 17345667209.23, 56478356209.345, 'some text as varchar209'); +INSERT INTO H2Table5 VALUES (210, 92233210, 17345667210.23, 56478356210.345, 'some text as varchar210'); + +#--- Created TABLE -5 : +CREATE TABLE H2Table6 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); + +#--- Inserted values in TABLE -5 : +INSERT INTO H2Table6 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); +INSERT INTO H2Table6 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); +INSERT INTO H2Table6 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); +INSERT INTO H2Table6 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); +INSERT INTO H2Table6 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); +INSERT INTO H2Table6 VALUES (212, 92233212, 17345667212.23, 56478356212.345, 'some text as varchar212'); + +#----------------------------------------------------------------------------------------------------------------------------------- + + +#---- JOINS QUERIES +SELECT * from H2Table4 AS T4 INNER JOIN H2Table6 AS T6 ON T4.int_T4field1 = T6.int_T4field1; +SELECT * from H2Table4 AS T4 RIGHT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; +SELECT * from H2Table4 AS T4 LEFT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; +SELECT * from H2Table4 CROSS JOIN H2Table6; +SELECT * from H2Table6 CROSS JOIN H2Table5; + +#---- SET/UNION OPERATIONS QUERIES +SELECT int_T4field1 FROM H2Table4 UNION SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 UNION ALL SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 INTERSECT SELECT int_T4field1 FROM H2Table6; +SELECT int_T4field1 FROM H2Table4 MINUS SELECT int_T4field1 FROM H2Table6; + +#---- ORDER BY CLAUSE QUERIES +SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 ASC; +SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 DESC; + +#---- GROUP BY CLAUSE QUERIES +SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2; + +#---- HAVING CLAUSE QUERIES +SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2 +HAVING h2t1.bool_field2 IS TRUE AND h2t1.tinyint_field3 > 41; + +#---- CASE STATEMENT QUERY +SELECT int_field1, BOOL_FIELD2, tinyint_field3, CASE WHEN tinyint_field3 > 44 THEN TRUE ELSE FALSE END FROM H2Table1; + +#---- AGGREGATE FUNCTIONS QUERIES +SELECT MAX(int_field1) AS Int_Val, MAX(tinyint_field3) AS TinyInt_Val, MAX(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT MIN(int_field1) AS Int_Val, MIN(tinyint_field3) AS TinyInt_Val, MIN(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT AVG(int_field1) AS Int_Val, AVG(tinyint_field3) AS TinyInt_Val, AVG(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT SUM(int_field1) AS Int_Val, SUM(tinyint_field3) AS TinyInt_Val, SUM(smallint_field4) AS SmallInt_Val FROM H2Table1; +SELECT COUNT(int_field1) AS Int_Val, COUNT(tinyint_field3) AS TinyInt_Val, COUNT(smallint_field4) AS SmallInt_Val FROM H2Table1; + +SELECT char_field16 FROM H2Table1; +SELECT UCASE (char_field16) AS Char_Val, UCASE (varchar_field13) AS VarChar_Val FROM H2Table1; +SELECT LCASE (char_field16) AS Char_Val, LCASE (varchar_field13) AS VarChar_Val FROM H2Table1; + +SELECT ROUND (decimal_field6, 2) AS Decimal_Val, ROUND (float_field21, 1) Float_Val, +ROUND (double_field7, 3) AS Double_Val, ROUND (real_field8, 4) AS Real_Val FROM H2Table1; + From a68cc231a01fd56156c95910390874ebe38ad5cd Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:26:17 -0400 Subject: [PATCH 106/129] file committed for review comment implementation --- .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java new file mode 100644 index 00000000000..eee69ea1bc3 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; + +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * + * JUnit Test class to test null values stored into Arrow vector for various datatypes for H2 database + * + */ +@RunWith(Parameterized.class) +public class JdbcToArrowNullTest { + + private Connection conn = null; + private Table table; + + private static final String NULL = "null"; + private static final String SELECTED_NULL_COLUMN = "selected_null_column"; + + private static final String[] testFiles = { + "h2/test1_all_datatypes_null_h2.yml", + "h2/test1_selected_datatypes_null_h2.yml" + }; + + /** + * Constructor which populate table object for each test iteration + * @param table + */ + public JdbcToArrowNullTest (Table table) { + this.table = table; + } + + /** + * This method creates Table object after reading YAML file + * @param ymlFilePath + * @return + * @throws IOException + */ + private static Table getTable(String ymlFilePath) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + /** + * Clean up method to close connection after test completes + * @throws SQLException + */ + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile: testFiles) { + tableArr[i++] = new Object[]{getTable(testFile)}; + } + return Arrays.asList(tableArr); + } + + /** + * This method tests null values stored into Arrow vector for various datatypes for H2 database + * @throws SQLException + * @throws IOException + */ + @Test + public void testNullValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + switch (table.getType()) { + case NULL: + sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); + break; + case SELECTED_NULL_COLUMN: + sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount()); + break; + } + } + } + + /** + * This method assert tests null values in vectors for all the datatypes + * @param vectors + * @param root + * @param rowCount + */ + public void sqlToArrowTestNullValues(String[] vectors, VectorSchemaRoot root, int rowCount) { + assertNullValues((IntVector)root.getVector(vectors[0]), rowCount); + assertNullValues((BitVector)root.getVector(vectors[1]), rowCount); + assertNullValues((TinyIntVector)root.getVector(vectors[2]), rowCount); + assertNullValues((SmallIntVector)root.getVector(vectors[3]), rowCount); + assertNullValues((BigIntVector)root.getVector(vectors[4]), rowCount); + assertNullValues((DecimalVector)root.getVector(vectors[5]), rowCount); + assertNullValues((Float8Vector)root.getVector(vectors[6]), rowCount); + assertNullValues((Float4Vector)root.getVector(vectors[7]), rowCount); + assertNullValues((TimeMilliVector)root.getVector(vectors[8]), rowCount); + assertNullValues((DateMilliVector)root.getVector(vectors[9]), rowCount); + assertNullValues((TimeStampVector)root.getVector(vectors[10]), rowCount); + assertNullValues((VarBinaryVector)root.getVector(vectors[11]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[12]), rowCount); + assertNullValues((VarBinaryVector)root.getVector(vectors[13]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[14]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[15]), rowCount); + assertNullValues((BitVector)root.getVector(vectors[16]), rowCount); + } + + /** + * This method assert tests null values in vectors for some selected datatypes + * @param vectors + * @param root + * @param rowCount + */ + public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSchemaRoot root, int rowCount) { + assertNullValues((BigIntVector)root.getVector(vectors[0]), rowCount); + assertNullValues((DecimalVector)root.getVector(vectors[1]), rowCount); + assertNullValues((Float8Vector)root.getVector(vectors[2]), rowCount); + assertNullValues((Float4Vector)root.getVector(vectors[3]), rowCount); + assertNullValues((TimeMilliVector)root.getVector(vectors[4]), rowCount); + assertNullValues((DateMilliVector)root.getVector(vectors[5]), rowCount); + assertNullValues((TimeStampVector)root.getVector(vectors[6]), rowCount); + assertNullValues((VarBinaryVector)root.getVector(vectors[7]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[8]), rowCount); + assertNullValues((VarBinaryVector)root.getVector(vectors[9]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[10]), rowCount); + assertNullValues((VarCharVector)root.getVector(vectors[11]), rowCount); + assertNullValues((BitVector)root.getVector(vectors[12]), rowCount); + } +} From 60314c65feafa0c006cce650c3dac77859d37ff5 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 06:26:35 -0400 Subject: [PATCH 107/129] file committed for review comment implementation --- .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java new file mode 100644 index 00000000000..595ddabf04b --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; + + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.TimeZone; + +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * + * JUnit Test class to test TimeZone based Date, Time and Timestamp datatypes for H2 database + * + */ +@RunWith(Parameterized.class) +public class JdbcToArrowTimeZoneTest { + + private Connection conn = null; + private Table table; + + private static final String EST_DATE = "est_date"; + private static final String EST_TIME = "est_time"; + private static final String EST_TIMESTAMP = "est_timestamp"; + private static final String GMT_DATE = "gmt_date"; + private static final String GMT_TIME = "gmt_time"; + private static final String GMT_TIMESTAMP = "gmt_timestamp"; + private static final String PST_DATE = "pst_date"; + private static final String PST_TIME = "pst_time"; + private static final String PST_TIMESTAMP = "pst_timestamp"; + + private static final String[] testFiles = { + "h2/test1_est_date_h2.yml", + "h2/test1_est_time_h2.yml", + "h2/test1_est_timestamp_h2.yml", + "h2/test1_gmt_date_h2.yml", + "h2/test1_gmt_time_h2.yml", + "h2/test1_gmt_timestamp_h2.yml", + "h2/test1_pst_date_h2.yml", + "h2/test1_pst_time_h2.yml", + "h2/test1_pst_timestamp_h2.yml" + }; + + /** + * Constructor which populate table object for each test iteration + * @param table + */ + public JdbcToArrowTimeZoneTest (Table table) { + this.table = table; + } + + /** + * This method creates Table object after reading YAML file + * @param ymlFilePath + * @return + * @throws IOException + */ + private static Table getTable(String ymlFilePath) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), + Table.class); + } + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + /** + * Clean up method to close connection after test completes + * @throws SQLException + */ + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile: testFiles) { + tableArr[i++] = new Object[]{getTable(testFile)}; + } + return Arrays.asList(tableArr); + } + + /** + * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database + * @throws SQLException + * @throws IOException + */ + @Test + public void testTimeZoneBasedValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { + switch (table.getType()) { + case EST_DATE: case GMT_DATE: case PST_DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIME: case GMT_TIME: case PST_TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIMESTAMP: case GMT_TIMESTAMP: case PST_TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + } + } + } + +} From 32ba284811fa2c420029a67eb9e36f1a7f8a0659 Mon Sep 17 00:00:00 2001 From: yashpal Date: Wed, 9 May 2018 08:11:42 -0400 Subject: [PATCH 108/129] Files committed to resolve compilation error --- .../adapter/jdbc/h2/JdbcToArrowTest.java | 45 +++++++++---------- .../jdbc/h2/JdbcToArrowTestH2Datatypes.java | 21 ++++++--- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index f7ef85068ff..22f9030c6f0 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -17,10 +17,9 @@ */ package org.apache.arrow.adapter.jdbc.h2; - +/* import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; @@ -38,27 +37,25 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; - import org.junit.After; import org.junit.Before; import org.junit.Test; - import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; - import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; +*/ +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; /** * */ public class JdbcToArrowTest extends AbstractJdbcToArrowTest { - private Connection conn = null; +/* private Connection conn = null; private ObjectMapper mapper = null; - - @Before + @Before public void setUp() throws Exception { String url = "jdbc:h2:mem:JdbcToArrowTest"; String driver = "org.h2.Driver"; @@ -122,7 +119,7 @@ public void sqlToArrowTestBool() throws Exception { int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); } catch (Exception e) { e.printStackTrace(); @@ -148,7 +145,7 @@ public void sqlToArrowTestTinyInt() throws Exception { int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); } catch (Exception e) { e.printStackTrace(); @@ -201,7 +198,7 @@ public void sqlToArrowTestBigInt() throws Exception { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 }; - assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); + assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); } catch (Exception e) { e.printStackTrace(); @@ -241,7 +238,7 @@ public void sqlToArrowTestBlob() throws Exception { hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") }; - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); } catch (Exception e) { e.printStackTrace(); @@ -268,7 +265,7 @@ public void sqlToArrowTestClobs() throws Exception { byte[][] varchars = { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); + assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); } catch (Exception e) { e.printStackTrace(); } finally { @@ -293,22 +290,22 @@ public void sqlToArrowTestAllDataTypes() throws Exception { int[] ints = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 }; - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, table.getInts()); + assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, table.getInts()); int[] bools = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); + assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); int[] tinyints = { 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 }; - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); + assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); int[] smallints = { 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 }; - assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); + assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); int[] bigints = { 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, @@ -330,20 +327,20 @@ public void sqlToArrowTestAllDataTypes() throws Exception { 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 }; - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 15, doubles); + assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 15, doubles); float[] reals = { 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f }; - assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), 15, reals); + assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), 15, reals); long[] times = { 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000 }; - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 15, times); + assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 15, times); long[] dates = { 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, @@ -374,15 +371,15 @@ public void sqlToArrowTestAllDataTypes() throws Exception { hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") }; - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 15, bytes); + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 15, bytes); byte[] strb = "some text that needs to be converted to varchar".getBytes(); byte[][] varchars = { strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb }; - assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 15, varchars); + assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 15, varchars); - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); + assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); strb = "some text that needs to be converted to clob".getBytes(); varchars = new byte[][] { @@ -404,5 +401,5 @@ public void sqlToArrowTestAllDataTypes() throws Exception { } } - +*/ } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java index e5b752fbc48..1b7814139d2 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java @@ -18,18 +18,19 @@ package org.apache.arrow.adapter.jdbc.h2; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +/* import org.junit.runners.Parameterized.Parameters; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.arrow.adapter.jdbc.Table; import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; -import java.util.TimeZone; +import java.util.Calendar; +import java.util.TimeZone; import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -45,18 +46,21 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import java.math.BigDecimal; import org.junit.After; import org.junit.Before; +import java.sql.Statement; + import org.junit.Test; import java.io.IOException; -import java.math.BigDecimal; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Statement; + import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; +//import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; @@ -69,9 +73,10 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; - +*/ @RunWith(Parameterized.class) public class JdbcToArrowTestH2Datatypes { + /* private static Connection conn = null; private static Table table; private int [] intValues; @@ -429,5 +434,7 @@ private static String join(String delimeter, String... columns) { } return builder.toString(); } + + */ } From 3a2b3120ae36ef5ae8f1118776a159a288575757 Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 10 May 2018 14:24:16 -0400 Subject: [PATCH 109/129] files committed based on atul's comment --- .../adapter/jdbc/h2/JdbcToArrowTest.java | 564 ++++++++---------- .../jdbc/h2/JdbcToArrowTestH2Datatypes.java | 440 -------------- .../resources/h2/test1_all_datatypes_h2.yml | 397 ++---------- 3 files changed, 283 insertions(+), 1118 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 22f9030c6f0..c5f14f98e34 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -17,7 +17,7 @@ */ package org.apache.arrow.adapter.jdbc.h2; -/* + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.arrow.adapter.jdbc.JdbcToArrow; @@ -40,366 +40,268 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; + import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; -*/ -import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; + /** * */ -public class JdbcToArrowTest extends AbstractJdbcToArrowTest { - -/* private Connection conn = null; - private ObjectMapper mapper = null; - - @Before - public void setUp() throws Exception { +@RunWith(Parameterized.class) +public class JdbcToArrowTest { + + private Connection conn = null; + private Table table; + + private static final String BIGINT = "BIGINT_FIELD5"; + private static final String BINARY = "BINARY_FIELD12"; + private static final String BIT = "BIT_FIELD17"; + private static final String BLOB = "BLOB_FIELD14"; + private static final String BOOL = "BOOL_FIELD2"; + private static final String CHAR = "CHAR_FIELD16"; + private static final String CLOB = "CLOB_FIELD15"; + private static final String DATE = "DATE_FIELD10"; + private static final String DECIMAL = "DECIMAL_FIELD6"; + private static final String DOUBLE = "DOUBLE_FIELD7"; + private static final String INT = "INT_FIELD1"; + private static final String REAL = "REAL_FIELD8"; + private static final String SMALLINT = "SMALLINT_FIELD4"; + private static final String TIME = "TIME_FIELD9"; + private static final String TIMESTAMP = "TIMESTAMP_FIELD11"; + private static final String TINYINT = "TINYINT_FIELD3"; + private static final String VARCHAR = "VARCHAR_FIELD13"; + + private static final String[] testFiles = {"h2/test1_all_datatypes_h2.yml"}; + + /** + * Constructor which populate table object for each test iteration + * @param table + */ + public JdbcToArrowTest(Table table) { + this.table = table; + } + + /** + * This method creates Table object after reading YAML file + * @param ymlFilePath + * @return + * @throws IOException + */ + private static Table getTable(String ymlFilePath) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( + JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + } + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { String url = "jdbc:h2:mem:JdbcToArrowTest"; String driver = "org.h2.Driver"; - - mapper = new ObjectMapper(new YAMLFactory()); - Class.forName(driver); - conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } } - + + /** + * Clean up method to close connection after test completes + * @throws SQLException + */ @After - public void destroy() throws Exception { + public void destroy() throws SQLException { if (conn != null) { conn.close(); conn = null; } } - - @Test - public void sqlToArrowTestInt() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_int_h2.yml"), - Table.class); - - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] values = { - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - }; - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, values); - - } catch (Exception e) { - throw e; - } finally { - deleteTestData(conn, table); - } - - } - - @Test - public void sqlToArrowTestBool() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_bool_h2.yml"), - Table.class); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] bools = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile: testFiles) { + tableArr[i++] = new Object[]{getTable(testFile)}; } - + return Arrays.asList(tableArr); } - + + /** + * This method tests various datatypes converted into Arrow vector for H2 database + * @throws SQLException + * @throws IOException + */ @Test - public void sqlToArrowTestTinyInt() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_tinyint_h2.yml"), - Table.class); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] tinyints = { - 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 - }; - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); + public void testDBValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), + getLongValues(table.getValues(), BIGINT)); + + assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), + getIntValues(table.getValues(), TINYINT)); + + assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), + getIntValues(table.getValues(), SMALLINT)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), + getBinaryValues(table.getValues(), BINARY)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), + getBinaryValues(table.getValues(), BLOB)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + getCharArrays(table.getValues(), CLOB)); + + assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + getCharArrays(table.getValues(), VARCHAR)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + getCharArrays(table.getValues(), CHAR)); + + assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), + getIntValues(table.getValues(), INT)); + + assertBitVectorValues((BitVector) root.getVector( BIT), table.getRowCount(), + getIntValues(table.getValues(), BIT)); + + assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), + getBooleanValues(table.getValues(), BOOL)); + + assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), + getLongValues(table.getValues(), DATE)); + + assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), + getLongValues(table.getValues(), TIME)); + + assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), + getLongValues(table.getValues(), TIMESTAMP)); + + assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), + getDecimalValues(table.getValues(), DECIMAL)); + + assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), + getDoubleValues(table.getValues(), DOUBLE)); + + assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), + getFloatValues(table.getValues(), REAL)); } - } - - @Test - public void sqlToArrowTestSmallInt() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_smallint_h2.yml"), - Table.class); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] smallints = { - 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 - }; - assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - + + private Integer [] getIntValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Integer [] valueArr = new Integer [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Integer.parseInt(data); + } + return valueArr; } - - @Test - public void sqlToArrowTestBigInt() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_bigint_h2.yml"), - Table.class); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] bigints = { - 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, - 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 - }; - assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - + + private Boolean [] getBooleanValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Boolean [] valueArr = new Boolean [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().equals("1"); + } + return valueArr; } - @Test - public void sqlToArrowTestBlob() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_blob_h2.yml"), - Table.class); - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - byte[][] bytes = { - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") - }; - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - + private BigDecimal [] getDecimalValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + BigDecimal [] valueArr = new BigDecimal [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr[i++] = new BigDecimal(data); + } + return valueArr; } - @Test - public void sqlToArrowTestClobs() throws Exception { - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_clob_h2.yml"), - Table.class); - - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - byte[] strb = "some text that needs to be converted to clob".getBytes(); - byte[][] varchars = { - strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb - }; - assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } + private Double [] getDoubleValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Double [] valueArr = new Double [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Double.parseDouble(data); + } + return valueArr; + } + private Float [] getFloatValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Float [] valueArr = new Float [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Float.parseFloat(data); + } + return valueArr; + } + private Long [] getLongValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Long [] valueArr = new Long [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Long.parseLong(data); + } + return valueArr; } - @Test - public void sqlToArrowTestAllDataTypes() throws Exception { - - Table table = - mapper.readValue( - this.getClass().getClassLoader().getResourceAsStream("h2/test1_all_datatypes_h2.yml"), - Table.class); - - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - try { - createTestData(conn, table); - - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), rootAllocator); - - int[] ints = { - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101 - }; - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), 15, table.getInts()); - - int[] bools = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), 15, bools); - - int[] tinyints = { - 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45 - }; - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 15, tinyints); - - int[] smallints = { - 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000 - }; - assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 15, smallints); - - int[] bigints = { - 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, 92233720, - 92233720, 92233720, 92233720, 92233720, 92233720, 92233720 - }; - assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 15, bigints); - - BigDecimal[] bigdecimals = { - new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), - new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), - new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), - new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), - new BigDecimal(17345667789.23), new BigDecimal(17345667789.23), new BigDecimal(17345667789.23) - }; - assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 15, bigdecimals); - - double[] doubles = { - 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, - 56478356785.345, 56478356785.345, 56478356785.345, - 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345, 56478356785.345 - }; - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 15, doubles); - - float[] reals = { - 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, - 56478356785.345f, 56478356785.345f, 56478356785.345f, - 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f, 56478356785.345f - }; - assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), 15, reals); - - long[] times = { - 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, - 45935000, 45935000, 45935000, 45935000, 45935000, 45935000, 45935000 - }; - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 15, times); - - long[] dates = { - 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, - 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l, 1518393600000l - }; - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), 15, dates); - - long[] timestamps = { - 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, - 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l, 1518439535000l - }; - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 15, timestamps); - - byte[][] bytes = { - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279"), - hexStringToByteArray("736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279") - }; - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 15, bytes); - - byte[] strb = "some text that needs to be converted to varchar".getBytes(); - byte[][] varchars = { - strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb - }; - assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 15, varchars); - - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 15, bytes); - - strb = "some text that needs to be converted to clob".getBytes(); - varchars = new byte[][] { - strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb - }; - assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), 15, varchars); - - strb = "some char text".getBytes(); - varchars = new byte[][] { - strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb, strb - }; - assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), 15, varchars); - - assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), 15, bools); - } catch (Exception e) { - e.printStackTrace(); - } finally { - deleteTestData(conn, table); - } - + private byte [][] getCharArrays(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().getBytes(); + } + return valueArr; + } + + private byte [][] getBinaryValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = hexStringToByteArray(data.trim()); + } + return valueArr; + } + + private String [] getValues(String[] values, String dataType) { + String value = ""; + for(String val : values) { + if(val.startsWith(dataType)) { + value = val.split("=")[1]; + break; + } + } + return value.split(","); } -*/ } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java deleted file mode 100644 index 1b7814139d2..00000000000 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTestH2Datatypes.java +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.adapter.jdbc.h2; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -/* -import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.arrow.adapter.jdbc.Table; -import java.util.Arrays; -import java.util.Collection; - -import java.util.Calendar; -import java.util.TimeZone; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeStampVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import java.math.BigDecimal; -import org.junit.After; -import org.junit.Before; -import java.sql.Statement; - -import org.junit.Test; -import java.io.IOException; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - - -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; -//import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitBooleanVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; -*/ -@RunWith(Parameterized.class) -public class JdbcToArrowTestH2Datatypes { - /* - private static Connection conn = null; - private static Table table; - private int [] intValues; - private int [] boolValues; - private BigDecimal [] decimalValues; - private double [] doubleValues; - private int [] tinyIntValues; - private int [] smallIntValues; - private int [] bigIntValues; - private long [] timeValues; - private long [] dateValues; - private long [] timestampValues; - private float [] realValues; - private byte [][] byteValues; - private byte [][] varCharValues; - private byte [][] charValues; - private byte [][] clobValues; - private long [] pstTimeValues; - private long [] estTimeValues; - private long [] gmtTimeValues; - private long [] pstDateValues; - private long [] estDateValues; - private long [] gmtDateValues; - private long [] pstTimestampValues; - private long [] estTimestampValues; - private long [] gmtTimestampValues; - - public JdbcToArrowTestH2Datatypes (Table data) { - super(); - this.intValues = data.getInts(); - this.boolValues = data.getBooleans(); - this.decimalValues = data.getDecimals(); - this.doubleValues = data.getDoubles(); - this.tinyIntValues = data.getTinyInts(); - this.smallIntValues = data.getSmallInts(); - this.bigIntValues = data.getBigInts(); - this.timeValues = data.getTimes(); - this.dateValues = data.getDates(); - this.timestampValues = data.getTimestamps(); - this.realValues = data.getReals(); - this.byteValues = data.getHexStringAsByte(); - this.varCharValues = data.getVarCharAsByte(); - this.charValues = data.getCharAsByte(); - this.clobValues = data.getClobAsByte(); - this.pstTimeValues = data.getPstTime(); - this.estTimeValues = data.getEstTime(); - this.gmtTimeValues = data.getGmtTime(); - this.pstDateValues = data.getPstDate(); - this.estDateValues = data.getEstDate(); - this.gmtDateValues = data.getGmtDate(); - this.pstTimestampValues = data.getPstTimestamp(); - this.estTimestampValues = data.getEstTimestamp() ; - this.gmtTimestampValues = data.getGmtTimestamp(); - - } - - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, JsonMappingException, JsonParseException, IOException { - setUp(); - return Arrays.asList(new Object[][]{{table}}); - } - - @Test - public void testDBValues() { - try { - sqlToArrowTestInt(); - sqlToArrowTestBool(); - sqlToArrowTestTinyInts(); - sqlToArrowTestSmallInts(); - sqlToArrowTestBigInts(); - sqlToArrowTestBigDecimals(); - sqlToArrowTestDoubles(); - sqlToArrowTestRealValues(); - sqlToArrowTestTimeValues(); - sqlToArrowTestDateValues(); - sqlToArrowTestTimestampValues(); - sqlToArrowTestByteValues(); - sqlToArrowTestVarCharValues(); - sqlToArrowTestCharValues(); - sqlToArrowTestBlobValues(); - sqlToArrowTestClobValues(); - sqlToArrowTestBits(); - sqlToArrowTestNullValues(); - sqlToArrowTestValuesWithPSTTimeZone(); - sqlToArrowTestValuesWithESTTimeZone(); - sqlToArrowTestValuesWithGMTTimeZone(); - sqlToArrowTestSelectedColumnsNullValues(); - } catch (SQLException sqe) { - sqe.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void sqlToArrowTestInt() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("INT_FIELD1 = 101", true, "INT_FIELD1"), - new RootAllocator(Integer.MAX_VALUE))){ - assertIntVectorValues((IntVector)root.getVector("INT_FIELD1"), intValues.length, intValues); - } - } - - public void sqlToArrowTestBool() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("BOOL_FIELD2 = 1", true, "BOOL_FIELD2"), - new RootAllocator(Integer.MAX_VALUE))){ - assertBitBooleanVectorValues((BitVector)root.getVector("BOOL_FIELD2"), boolValues.length, boolValues); - } - } - - public void sqlToArrowTestTinyInts() throws SQLException, IOException { - try(VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("TINYINT_FIELD3 = 45", true, "TINYINT_FIELD3"), - new RootAllocator(Integer.MAX_VALUE))){ - assertTinyIntVectorValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), tinyIntValues.length, tinyIntValues); - } - } - - public void sqlToArrowTestSmallInts() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery("SMALLINT_FIELD4 = 12000", true, "SMALLINT_FIELD4"), - new RootAllocator(Integer.MAX_VALUE))){ - assertSmallIntVectorValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), smallIntValues.length, smallIntValues); - } - } - - public void sqlToArrowTestBigInts() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIGINT_FIELD5"), - new RootAllocator(Integer.MAX_VALUE))){ - assertBigIntVectorValues((BigIntVector)root.getVector("BIGINT_FIELD5"), bigIntValues.length, bigIntValues); - } - } - - public void sqlToArrowTestBigDecimals() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DECIMAL_FIELD6"), - new RootAllocator(Integer.MAX_VALUE))){ - assertDecimalVectorValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), decimalValues.length, decimalValues); - } - } - - public void sqlToArrowTestDoubles() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DOUBLE_FIELD7"), - new RootAllocator(Integer.MAX_VALUE))){ - assertFloat8VectorValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), doubleValues.length, doubleValues); - } - } - - public void sqlToArrowTestRealValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "REAL_FIELD8"), - new RootAllocator(Integer.MAX_VALUE))){ - assertFloat4VectorValues((Float4Vector)root.getVector("REAL_FIELD8"), realValues.length, realValues); - } - } - - public void sqlToArrowTestTimeValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIME_FIELD9"), - new RootAllocator(Integer.MAX_VALUE))){ - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), timeValues.length, timeValues); - } - } - - public void sqlToArrowTestDateValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "DATE_FIELD10"), - new RootAllocator(Integer.MAX_VALUE))){ - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), dateValues.length, dateValues); - } - } - - public void sqlToArrowTestTimestampValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "TIMESTAMP_FIELD11"), - new RootAllocator(Integer.MAX_VALUE))){ - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), timestampValues.length, timestampValues); - } - } - - public void sqlToArrowTestByteValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BINARY_FIELD12"), - new RootAllocator(Integer.MAX_VALUE))){ - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), byteValues.length, byteValues); - - } - } - - public void sqlToArrowTestVarCharValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "VARCHAR_FIELD13"), - new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), varCharValues.length, varCharValues); - } - } - - public void sqlToArrowTestBlobValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BLOB_FIELD14"), - new RootAllocator(Integer.MAX_VALUE))){ - assertVarBinaryVectorValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), byteValues.length, byteValues); - } - } - - public void sqlToArrowTestClobValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CLOB_FIELD15"), - new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("CLOB_FIELD15"), clobValues.length, clobValues); - } - } - - public void sqlToArrowTestCharValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "CHAR_FIELD16"), - new RootAllocator(Integer.MAX_VALUE))){ - assertVarcharVectorValues((VarCharVector)root.getVector("CHAR_FIELD16"), charValues.length, charValues); - } - } - - public void sqlToArrowTestBits() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery(true, "BIT_FIELD17"), - new RootAllocator(Integer.MAX_VALUE))){ - assertBitBooleanVectorValues((BitVector)root.getVector("BIT_FIELD17"), boolValues.length, boolValues); - } - } - - public void sqlToArrowTestNullValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, - getQuery (false, table.getAllColumns()), new RootAllocator(Integer.MAX_VALUE))){ - assertNullValues((IntVector)root.getVector("INT_FIELD1"), 5); - assertNullValues((BitVector)root.getVector("BOOL_FIELD2"), 5); - assertNullValues((TinyIntVector)root.getVector("TINYINT_FIELD3"), 5); - assertNullValues((SmallIntVector)root.getVector("SMALLINT_FIELD4"), 5); - assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); - assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); - assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); - assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); - assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); - assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); - assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); - assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); - assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); - assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); - assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); - assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); - assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); - } - } - - public void sqlToArrowTestValuesWithPSTTimeZone() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, - getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("PST")))){ - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), pstTimeValues.length, pstTimeValues); - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), pstDateValues.length, pstDateValues); - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), pstTimestampValues.length, - pstTimestampValues); - } - } - - public void sqlToArrowTestValuesWithESTTimeZone() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, - getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("EST")))){ - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), estTimeValues.length, estTimeValues); - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), estDateValues.length, estDateValues); - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), estTimestampValues.length, - estTimestampValues); - } - } - - public void sqlToArrowTestValuesWithGMTTimeZone() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, - getQuery(" rownum < 6 ", true, "TIME_FIELD9", "DATE_FIELD10", "TIMESTAMP_FIELD11"), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone("GMT")))){ - assertTimeVectorValues((TimeMilliVector)root.getVector("TIME_FIELD9"), gmtTimeValues.length, gmtTimeValues); - assertDateVectorValues((DateMilliVector)root.getVector("DATE_FIELD10"), gmtDateValues.length, gmtDateValues); - assertTimeStampVectorValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), gmtTimestampValues.length, - gmtTimestampValues); - } - } - - public void sqlToArrowTestSelectedColumnsNullValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, getQuery ("INT_FIELD1 = 102", true, table.getAllColumns()), - new RootAllocator(Integer.MAX_VALUE))){ - assertNullValues((BigIntVector)root.getVector("BIGINT_FIELD5"), 5); - assertNullValues((DecimalVector)root.getVector("DECIMAL_FIELD6"), 5); - assertNullValues((Float8Vector)root.getVector("DOUBLE_FIELD7"), 5); - assertNullValues((Float4Vector)root.getVector("REAL_FIELD8"), 5); - assertNullValues((TimeMilliVector)root.getVector("TIME_FIELD9"), 5); - assertNullValues((DateMilliVector)root.getVector("DATE_FIELD10"), 5); - assertNullValues((TimeStampVector)root.getVector("TIMESTAMP_FIELD11"), 5); - assertNullValues((VarBinaryVector)root.getVector("BINARY_FIELD12"), 5); - assertNullValues((VarCharVector)root.getVector("VARCHAR_FIELD13"), 5); - assertNullValues((VarBinaryVector)root.getVector("BLOB_FIELD14"), 5); - assertNullValues((VarCharVector)root.getVector("CLOB_FIELD15"), 5); - assertNullValues((VarCharVector)root.getVector("CHAR_FIELD16"), 5); - assertNullValues((BitVector)root.getVector("BIT_FIELD17"), 5); - } - } - - @Before - public void createTestData() throws SQLException{ - try (Statement stmt = conn.createStatement();){ - stmt.executeUpdate(table.getCreate()); - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - } - } - - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } - - private static void setUp() throws SQLException, ClassNotFoundException, JsonMappingException, JsonParseException, IOException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - table = getTable ("h2/test1_all_datatypes_h2.yml"); - } - - private static Table getTable (String ymlFilePath) throws JsonMappingException, JsonParseException, IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowTestH2Datatypes.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } - - private String getQuery (boolean isNotNull, String... columns) { - return getQuery("", isNotNull, columns); - - } - - private String getQuery (String whereClause, boolean isNotNull, String... columns) { - return getQuery( whereClause, isNotNull, 0, columns); - - } - - private String getQuery(String whereClause, boolean isNotNull, int index, String... columns) { - StringBuilder query = new StringBuilder(String.format("select %s from %s ", -// columns.length > 1 ? String.join(",", columns) : columns[index], table.getName())); - columns.length > 1 ? join(",", columns) : columns[index], table.getName())); - query.append(whereClause != null && !whereClause.isEmpty() ? " where " + whereClause + " and " : " where "); - columns = columns.length == 1 && columns[0].contains(",") ? columns[0].split(",") : columns; - query.append(isNotNull ? columns[index] + " is not null;" : columns[index] + " is null;"); - - return query.toString(); - } - - private static String join(String delimeter, String... columns) { - if (columns.length == 1) { - return columns[0]; - } - StringBuilder builder = new StringBuilder(); - int i = 0; - for (String column: columns) { - if (i > 0) { - builder.append(delimeter).append(" "); - } - builder.append(column); - i++; - } - return builder.toString(); - } - - */ -} - diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index ef75f323f4d..06625422526 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -9,326 +9,6 @@ #OF ANY KIND, either express or implied. See the License for the specific #language governing permissions and limitations under the License. -ints: - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - - '101' - -booleans: - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - - '1' - -tinyInts: - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - - '45' - -smallInts: - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - - '12000' - -bigInts: - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - - '92233720' - -reals: - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - - '56478356785.345f' - -decimals: - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - - '17345667789.23' - -doubles: - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - - '56478356785.345' - -times: - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - -dates: - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - -timestamps: - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - -bytes: - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - - '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' - -varchars: - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - - 'some text that needs to be converted to varchar' - -clobs: - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - - 'some text that needs to be converted to clob' - -chars: - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - - 'some char text' - -pstTime: - - '56735000' - - '56735000' - - '56735000' - - '56735000' - - '56735000' - -estTime: - - '45935000' - - '45935000' - - '45935000' - - '45935000' - - '45935000' - -gmtTime: - - '27935000' - - '27935000' - - '27935000' - - '27935000' - - '27935000' - -pstDate: - - '1518336000000' - - '1518336000000' - - '1518336000000' - - '1518336000000' - - '1518336000000' - -estDate: - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - - '1518325200000' - -gmtDate: - - '1518307200000' - - '1518307200000' - - '1518307200000' - - '1518307200000' - - '1518307200000' - -pstTimestamp: - - '1518450335000' - - '1518450335000' - - '1518450335000' - - '1518450335000' - - '1518450335000' - -estTimestamp: - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - - '1518439535000' - -gmtTimestamp: - - '1518421535000' - - '1518421535000' - - '1518421535000' - - '1518421535000' - - '1518421535000' - -allColumns: 'int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17' - name: 'table1' create: 'CREATE TABLE table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, @@ -385,33 +65,56 @@ data: PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - - - 'INSERT INTO table1 VALUES (101, 1, 45, 12000, 92233720, 17345667789.23, 56478356785.345, 56478356785.345, PARSEDATETIME(''12:45:35 GMT'', ''HH:mm:ss z''), - PARSEDATETIME(''2018-02-12 GMT'', ''yyyy-MM-dd z''), PARSEDATETIME(''2018-02-12 12:45:35 GMT'', ''yyyy-MM-dd HH:mm:ss z''), - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to varchar'', - ''736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279'', ''some text that needs to be converted to clob'', ''some char text'', 1);' - + query: 'select int_field1, bool_field2, tinyint_field3, smallint_field4, bigint_field5, decimal_field6, double_field7, real_field8, time_field9, date_field10, timestamp_field11, binary_field12, varchar_field13, blob_field14, clob_field15, char_field16, bit_field17 from table1' -drop: 'DROP table table1;' \ No newline at end of file +drop: 'DROP table table1;' + +rowCount: '10' + +values: + - 'INT_FIELD1=101,101,101,101,101,101,101,101,101,101' + - 'BOOL_FIELD2=1,1,1,1,1,1,1,1,1,1' + - 'BIT_FIELD17=1,1,1,1,1,1,1,1,1,1' + - 'TINYINT_FIELD3=45,45,45,45,45,45,45,45,45,45' + - 'SMALLINT_FIELD4=12000,12000,12000,12000,12000,12000,12000,12000,12000,12000' + - 'BIGINT_FIELD5=92233720,92233720,92233720,92233720,92233720,92233720,92233720,92233720,92233720,92233720' + - 'REAL_FIELD8=56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f,56478356785.345f' + - 'DECIMAL_FIELD6=17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23' + - 'DOUBLE_FIELD7=56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345' + - 'TIME_FIELD9=45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000' + - 'DATE_FIELD10=1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000' + - 'TIMESTAMP_FIELD11=1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000' + - 'CHAR_FIELD16=some char text,some char text,some char text,some char text,some char text, + some char text,some char text,some char text,some char text,some char text' + - 'VARCHAR_FIELD13=some text that needs to be converted to varchar,some text that needs to be converted to varchar, + some text that needs to be converted to varchar,some text that needs to be converted to varchar, + some text that needs to be converted to varchar,some text that needs to be converted to varchar, + some text that needs to be converted to varchar,some text that needs to be converted to varchar, + some text that needs to be converted to varchar,some text that needs to be converted to varchar' + - 'BINARY_FIELD12=736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - 'BLOB_FIELD14=736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279, + 736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279' + - 'CLOB_FIELD15=some text that needs to be converted to clob,some text that needs to be converted to clob, + some text that needs to be converted to clob,some text that needs to be converted to clob, + some text that needs to be converted to clob,some text that needs to be converted to clob, + some text that needs to be converted to clob,some text that needs to be converted to clob, + some text that needs to be converted to clob,some text that needs to be converted to clob' \ No newline at end of file From 02952c4c1a9db7ff6247e4510320dbd0930b5ee6 Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 10 May 2018 15:09:04 -0400 Subject: [PATCH 110/129] File committed to resolve date error --- .../jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index 06625422526..03b3d3fed66 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -84,7 +84,7 @@ values: - 'DECIMAL_FIELD6=17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23' - 'DOUBLE_FIELD7=56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345' - 'TIME_FIELD9=45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000' - - 'DATE_FIELD10=1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000,1518325200000' + - 'DATE_FIELD10=1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000' - 'TIMESTAMP_FIELD11=1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000' - 'CHAR_FIELD16=some char text,some char text,some char text,some char text,some char text, some char text,some char text,some char text,some char text,some char text' From 99906161903b8e6bacbbaad504e4ab715ac8e674 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 10 May 2018 21:03:02 -0700 Subject: [PATCH 111/129] Removed unused file. --- .../src/test/resources/h2/testcases_h2.txt | 96 ------------------- 1 file changed, 96 deletions(-) delete mode 100644 java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt diff --git a/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt b/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt deleted file mode 100644 index 845b5ba717f..00000000000 --- a/java/adapter/jdbc/src/test/resources/h2/testcases_h2.txt +++ /dev/null @@ -1,96 +0,0 @@ -#--- Created TABLE -1 : -CREATE TABLE H2Table1 (int_field1 INT, bool_field2 BOOLEAN, tinyint_field3 TINYINT, smallint_field4 SMALLINT, bigint_field5 BIGINT, decimal_field6 DECIMAL(20,2), double_field7 DOUBLE, real_field8 REAL, time_field9 TIME, date_field10 DATE, timestamp_field11 TIMESTAMP, binary_field12 BINARY(100), varchar_field13 VARCHAR(256), blob_field14 BLOB, clob_field15 CLOB, char_field16 CHAR(16), bit_field17 BIT, identity_field18 IDENTITY, varchar_ignorecase_field19 VARCHAR_IGNORECASE(30), UUID_field20 UUID, float_field21 FLOAT); - -#--- Inserted values in TABLE -1 : -INSERT INTO H2Table1 VALUES (101, 1, 41, 12001, 92233721, 17345667781.23, 56478356781.345, 56478356781.345, '1:45:35', '2018-02-1', '2018-02-1 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as varchar1', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617271', 'some text as clob1', 'some char text1', 1, 10001, 'varchar_ignorecase_1', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'); -INSERT INTO H2Table1 VALUES (102, 0, 42, 12002, 92233722, 17345667782.23, 56478356782.345, 56478356782.345, '2:45:35', '2018-02-2', '2018-02-2 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as varchar2', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617272', 'some text as clob2', 'some char text2', 0, 10002, 'varchar_ignorecase_2', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'); -INSERT INTO H2Table1 VALUES (103, 1, 43, 12003, 92233723, 17345667783.23, 56478356783.345, 56478356783.345, '3:45:35', '2018-02-3', '2018-02-3 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as varchar3', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617273', 'some text as clob3', 'some char text3', 1, 10003, 'varchar_ignorecase_3', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13'); -INSERT INTO H2Table1 VALUES (104, 0, 44, 12004, 92233724, 17345667784.23, 56478356784.345, 56478356784.345, '4:45:35', '2018-02-4', '2018-02-4 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some as varchar4', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617274', 'some text as clob4', 'some char text4', 0, 10004, 'varchar_ignorecase_4', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14'); -INSERT INTO H2Table1 VALUES (105, 1, 45, 12005, 92233725, 17345667785.23, 56478356785.345, 56478356785.345, '5:45:35', '2018-02-5', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as varchar5', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob5', 'some char text5', 1, 10005, 'varchar_ignorecase_5', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15'); - -#--- Created TABLE -2 : -CREATE TABLE H2Table2 (int_T2field1 INT, bool_T2field2 BOOLEAN, tinyint_T2field3 TINYINT, smallint_T2field4 SMALLINT, bigint_T2field5 BIGINT, decimal_T2field6 DECIMAL(20,2), double_T2field7 DOUBLE, real_T2field8 REAL, time_T2field9 TIME, date_T2field10 DATE, timestamp_T2field11 TIMESTAMP, binary_T2field12 BINARY(100), varchar_T2field13 VARCHAR(256), blob_T2field14 BLOB, clob_T2field15 CLOB, char_T2field16 CHAR(16), bit_T2field17 BIT, identity_T2field18 IDENTITY, varchar_ignorecase_T2field19 VARCHAR_IGNORECASE(30), UUID_T2field20 UUID, float_T2field21 FLOAT); - -#--- Inserted values in TABLE -2 : -INSERT INTO H2Table2 VALUES (106, 1, 46, 12006, 92233726, 17345667786.23, 56478356786.345, 56478356786.345, '6:45:35', '2018-02-6', '2018-02-6 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as varchar6', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617276', 'some text as clob6', 'some char text6', 0, 10006, 'varchar_ignorecase_6', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16'); -INSERT INTO H2Table2 VALUES (107, 0, 47, 12007, 92233727, 17345667787.23, 56478356787.345, 56478356787.345, '7:45:35', '2018-02-7','2018-02-7 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as varchar7', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617277', 'some text as clob7', 'some char text7', 1, 10007, 'varchar_ignorecase_7', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17'); -INSERT INTO H2Table2 VALUES (108, 1, 48, 02008, 92233728, 17345667788.23, 56478356788.345, 56478356788.345, '8:45:35', '2018-02-8', '2018-02-5 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as varchar8', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617278', 'some text as clob8', 'some char text8', 1, 10008, 'varchar_ignorecase_8', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18'); -INSERT INTO H2Table2 VALUES (109, 0, 49, 12009, 92233729, 17345667789.23, 56478356789.345, 56478356789.345, '9:45:35', '2018-02-9', '2018-02-9 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as varchar9', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617279', 'some text as clob9', 'some char text9', 0, 10009, 'varchar_ignorecase_9', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a19'); -INSERT INTO H2Table2 VALUES (110, 1, 50, 12010, 92233710, 17345667710.23, 56478356710.345, 56478356710.345, '10:45:35', '2018-02-10', '2018-02-10 12:45:35', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617210', 'some text as varchar10', '736f6d6520746578742074686174206e6565647320746f20626520636f6e76657274656420746f2062696e617275', 'some text as clob10', 'some char text10', 1, 10010, 'varchar_ignorecase_10', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a10'); - -#--- Created TABLE -3 : -CREATE TABLE H2Table4 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); - ---- Inserted values in TABLE -3 : -INSERT INTO H2Table4 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); -INSERT INTO H2Table4 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); -INSERT INTO H2Table4 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); -INSERT INTO H2Table4 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); -INSERT INTO H2Table4 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); -INSERT INTO H2Table4 VALUES (214, 92233214, 17345667214.23, 56478356214.345, 'some text as varchar214'); - -#--- Created TABLE -4 : -CREATE TABLE H2Table5 (int_T5field1 INT, bigint_T5field2 BIGINT, decimal_T5field3 DECIMAL(20,2), float_T5field4 FLOAT, varchar_T5field5 VARCHAR(256)); - -#--- Inserted values in TABLE -4 : -INSERT INTO H2Table5 VALUES (206, 92233206, 17345667206.23, 56478356206.345, 'some text as varchar206'); -INSERT INTO H2Table5 VALUES (207, 92233207, 17345667207.23, 56478356207.345, 'some text as varchar207'); -INSERT INTO H2Table5 VALUES (208, 92233208, 17345667208.23, 56478356208.345, 'some text as varchar208'); -INSERT INTO H2Table5 VALUES (209, 92233209, 17345667209.23, 56478356209.345, 'some text as varchar209'); -INSERT INTO H2Table5 VALUES (210, 92233210, 17345667210.23, 56478356210.345, 'some text as varchar210'); - -#--- Created TABLE -5 : -CREATE TABLE H2Table6 (int_T4field1 INT, bigint_T4field2 BIGINT, decimal_T4field3 DECIMAL(20,2), float_T4field4 FLOAT, varchar_T4field5 VARCHAR(256)); - -#--- Inserted values in TABLE -5 : -INSERT INTO H2Table6 VALUES (201, 92233201, 17345667201.23, 56478356201.345, 'some text as varchar201'); -INSERT INTO H2Table6 VALUES (202, 92233202, 17345667202.23, 56478356202.345, 'some text as varchar202'); -INSERT INTO H2Table6 VALUES (203, 92233203, 17345667203.23, 56478356203.345, 'some text as varchar203'); -INSERT INTO H2Table6 VALUES (204, 92233204, 17345667204.23, 56478356204.345, 'some text as varchar204'); -INSERT INTO H2Table6 VALUES (205, 92233205, 17345667205.23, 56478356205.345, 'some text as varchar205'); -INSERT INTO H2Table6 VALUES (212, 92233212, 17345667212.23, 56478356212.345, 'some text as varchar212'); - -#----------------------------------------------------------------------------------------------------------------------------------- - - -#---- JOINS QUERIES -SELECT * from H2Table4 AS T4 INNER JOIN H2Table6 AS T6 ON T4.int_T4field1 = T6.int_T4field1; -SELECT * from H2Table4 AS T4 RIGHT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; -SELECT * from H2Table4 AS T4 LEFT JOIN H2Table5 AS T5 ON T4.int_T4field1 = T5.int_T5field1; -SELECT * from H2Table4 CROSS JOIN H2Table6; -SELECT * from H2Table6 CROSS JOIN H2Table5; - -#---- SET/UNION OPERATIONS QUERIES -SELECT int_T4field1 FROM H2Table4 UNION SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 UNION ALL SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 INTERSECT SELECT int_T4field1 FROM H2Table6; -SELECT int_T4field1 FROM H2Table4 MINUS SELECT int_T4field1 FROM H2Table6; - -#---- ORDER BY CLAUSE QUERIES -SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 ASC; -SELECT int_T4field1, bigint_T4field2, varchar_T4field5 FROM H2Table6 ORDER BY int_T4field1 DESC; - -#---- GROUP BY CLAUSE QUERIES -SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2; - -#---- HAVING CLAUSE QUERIES -SELECT h2t1.tinyint_field3, h2t1.bool_field2 FROM H2Table1 h2t1 GROUP BY h2t1.tinyint_field3, h2t1.bool_field2 -HAVING h2t1.bool_field2 IS TRUE AND h2t1.tinyint_field3 > 41; - -#---- CASE STATEMENT QUERY -SELECT int_field1, BOOL_FIELD2, tinyint_field3, CASE WHEN tinyint_field3 > 44 THEN TRUE ELSE FALSE END FROM H2Table1; - -#---- AGGREGATE FUNCTIONS QUERIES -SELECT MAX(int_field1) AS Int_Val, MAX(tinyint_field3) AS TinyInt_Val, MAX(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT MIN(int_field1) AS Int_Val, MIN(tinyint_field3) AS TinyInt_Val, MIN(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT AVG(int_field1) AS Int_Val, AVG(tinyint_field3) AS TinyInt_Val, AVG(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT SUM(int_field1) AS Int_Val, SUM(tinyint_field3) AS TinyInt_Val, SUM(smallint_field4) AS SmallInt_Val FROM H2Table1; -SELECT COUNT(int_field1) AS Int_Val, COUNT(tinyint_field3) AS TinyInt_Val, COUNT(smallint_field4) AS SmallInt_Val FROM H2Table1; - -SELECT char_field16 FROM H2Table1; -SELECT UCASE (char_field16) AS Char_Val, UCASE (varchar_field13) AS VarChar_Val FROM H2Table1; -SELECT LCASE (char_field16) AS Char_Val, LCASE (varchar_field13) AS VarChar_Val FROM H2Table1; - -SELECT ROUND (decimal_field6, 2) AS Decimal_Val, ROUND (float_field21, 1) Float_Val, -ROUND (double_field7, 3) AS Double_Val, ROUND (real_field8, 4) AS Real_Val FROM H2Table1; - From 52c004750371b61ae07d23d70af8413bf15b0828 Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Thu, 10 May 2018 21:04:03 -0700 Subject: [PATCH 112/129] Added qualified import stmts. Removed unused file. --- .../arrow/adapter/jdbc/h2/JdbcToArrowTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index c5f14f98e34..c6680af8921 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -54,7 +54,21 @@ import java.util.Calendar; import java.util.Collection; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBitVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBooleanVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDecimalVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat4VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertFloat8VectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertSmallIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.hexStringToByteArray; /** From 16cda229ec675539a8fd3ce1868aa7625baed2ad Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 22 May 2018 07:42:11 -0400 Subject: [PATCH 113/129] Files committed Charset and Resultset based testcases --- .../test/resources/h2/test1_charset_ch_h2.yml | 43 +++++++++++++++ .../test/resources/h2/test1_charset_h2.yml | 53 +++++++++++++++++++ .../test/resources/h2/test1_charset_jp_h2.yml | 43 +++++++++++++++ .../test/resources/h2/test1_charset_kr_h2.yml | 43 +++++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_charset_ch_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_charset_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_charset_jp_h2.yml create mode 100644 java/adapter/jdbc/src/test/resources/h2/test1_charset_kr_h2.yml diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_charset_ch_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_charset_ch_h2.yml new file mode 100644 index 00000000000..1b6d7d503b8 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_charset_ch_h2.yml @@ -0,0 +1,43 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'charset' + +vectors: + - 'VARCHAR_FIELD13' + - 'CHAR_FIELD16' + - 'CLOB_FIELD15' + +rowCount: '5' + +charSet: 'GBK' + +create: 'CREATE TABLE table1 (int_field1 INT, varchar_field13 VARCHAR(256), clob_field15 CLOB, char_field16 CHAR(128));' + +data: + - 'INSERT INTO table1 VALUES (101,''一些帶有char編碼的文本需要轉換為varchar'', ''一些带有char编码的文本需要转换为clob'', ''一些char编码的字符文本'');' + - 'INSERT INTO table1 VALUES (101,''一些帶有char編碼的文本需要轉換為varchar'', ''一些带有char编码的文本需要转换为clob'', ''一些char编码的字符文本'');' + - 'INSERT INTO table1 VALUES (101,''一些帶有char編碼的文本需要轉換為varchar'', ''一些带有char编码的文本需要转换为clob'', ''一些char编码的字符文本'');' + - 'INSERT INTO table1 VALUES (101,''一些帶有char編碼的文本需要轉換為varchar'', ''一些带有char编码的文本需要转换为clob'', ''一些char编码的字符文本'');' + - 'INSERT INTO table1 VALUES (101,''一些帶有char編碼的文本需要轉換為varchar'', ''一些带有char编码的文本需要转换为clob'', ''一些char编码的字符文本'');' + +query: 'select varchar_field13, clob_field15, char_field16 from table1' + +drop: 'DROP table table1;' + +values: + - 'VARCHAR_FIELD13=一些帶有char編碼的文本需要轉換為varchar,一些帶有char編碼的文本需要轉換為varchar,一些帶有char編碼的文本需要轉換為varchar, + 一些帶有char編碼的文本需要轉換為varchar,一些帶有char編碼的文本需要轉換為varchar' + - 'CLOB_FIELD15=一些带有char编码的文本需要转换为clob,一些带有char编码的文本需要转换为clob,一些带有char编码的文本需要转换为clob, + 一些带有char编码的文本需要转换为clob,一些带有char编码的文本需要转换为clob' + - 'CHAR_FIELD16=一些char编码的字符文本,一些char编码的字符文本,一些char编码的字符文本,一些char编码的字符文本,一些char编码的字符文本' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_charset_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_charset_h2.yml new file mode 100644 index 00000000000..75734a221b8 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_charset_h2.yml @@ -0,0 +1,53 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'charset' + +vectors: + - 'VARCHAR_FIELD13' + - 'CHAR_FIELD16' + - 'CLOB_FIELD15' + +rowCount: '10' + +create: 'CREATE TABLE table1 (int_field1 INT, varchar_field13 VARCHAR(256), clob_field15 CLOB, char_field16 CHAR(128));' + +data: + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + - 'INSERT INTO table1 VALUES (101,''some text with char encoding that needs to be converted to varchar'', ''some text with char encoding that needs to be converted to clob'', ''some char text with char encoding'');' + +query: 'select varchar_field13, clob_field15, char_field16 from table1' + +drop: 'DROP table table1;' + +values: + - 'VARCHAR_FIELD13=some text with char encoding that needs to be converted to varchar,some text with char encoding that needs to be converted to varchar, + some text with char encoding that needs to be converted to varchar,some text with char encoding that needs to be converted to varchar, + some text with char encoding that needs to be converted to varchar,some text with char encoding that needs to be converted to varchar, + some text with char encoding that needs to be converted to varchar,some text with char encoding that needs to be converted to varchar, + some text with char encoding that needs to be converted to varchar,some text with char encoding that needs to be converted to varchar' + - 'CLOB_FIELD15=some text with char encoding that needs to be converted to clob,some text with char encoding that needs to be converted to clob, + some text with char encoding that needs to be converted to clob,some text with char encoding that needs to be converted to clob, + some text with char encoding that needs to be converted to clob,some text with char encoding that needs to be converted to clob, + some text with char encoding that needs to be converted to clob,some text with char encoding that needs to be converted to clob, + some text with char encoding that needs to be converted to clob,some text with char encoding that needs to be converted to clob' + - 'CHAR_FIELD16=some char text with char encoding,some char text with char encoding,some char text with char encoding,some char text with char encoding,some char text with char encoding, + some char text with char encoding,some char text with char encoding,some char text with char encoding,some char text with char encoding,some char text with char encoding' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_charset_jp_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_charset_jp_h2.yml new file mode 100644 index 00000000000..10c33f443b6 --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_charset_jp_h2.yml @@ -0,0 +1,43 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'charset' + +vectors: + - 'VARCHAR_FIELD13' + - 'CHAR_FIELD16' + - 'CLOB_FIELD15' + +rowCount: '5' + +charSet: 'SJIS' + +create: 'CREATE TABLE table1 (int_field1 INT, varchar_field13 VARCHAR(256), clob_field15 CLOB, char_field16 CHAR(128));' + +data: + - 'INSERT INTO table1 VALUES (101,''varcharに変換する必要があるcharエンコーディングのテキスト'', ''charエンコーディングのあるテキストをclobに変換する必要がある'', ''charエンコーディングのあるcharテキスト'');' + - 'INSERT INTO table1 VALUES (101,''varcharに変換する必要があるcharエンコーディングのテキスト'', ''charエンコーディングのあるテキストをclobに変換する必要がある'', ''charエンコーディングのあるcharテキスト'');' + - 'INSERT INTO table1 VALUES (101,''varcharに変換する必要があるcharエンコーディングのテキスト'', ''charエンコーディングのあるテキストをclobに変換する必要がある'', ''charエンコーディングのあるcharテキスト'');' + - 'INSERT INTO table1 VALUES (101,''varcharに変換する必要があるcharエンコーディングのテキスト'', ''charエンコーディングのあるテキストをclobに変換する必要がある'', ''charエンコーディングのあるcharテキスト'');' + - 'INSERT INTO table1 VALUES (101,''varcharに変換する必要があるcharエンコーディングのテキスト'', ''charエンコーディングのあるテキストをclobに変換する必要がある'', ''charエンコーディングのあるcharテキスト'');' + +query: 'select varchar_field13, clob_field15, char_field16 from table1' + +drop: 'DROP table table1;' + +values: + - 'VARCHAR_FIELD13=varcharに変換する必要があるcharエンコーディングのテキスト,varcharに変換する必要があるcharエンコーディングのテキスト,varcharに変換する必要があるcharエンコーディングのテキスト, + varcharに変換する必要があるcharエンコーディングのテキスト,varcharに変換する必要があるcharエンコーディングのテキスト' + - 'CLOB_FIELD15=charエンコーディングのあるテキストをclobに変換する必要がある,charエンコーディングのあるテキストをclobに変換する必要がある,charエンコーディングのあるテキストをclobに変換する必要がある, + charエンコーディングのあるテキストをclobに変換する必要がある,charエンコーディングのあるテキストをclobに変換する必要がある' + - 'CHAR_FIELD16=charエンコーディングのあるcharテキスト,charエンコーディングのあるcharテキスト,charエンコーディングのあるcharテキスト,charエンコーディングのあるcharテキスト,charエンコーディングのあるcharテキスト' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_charset_kr_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_charset_kr_h2.yml new file mode 100644 index 00000000000..a00a41b539a --- /dev/null +++ b/java/adapter/jdbc/src/test/resources/h2/test1_charset_kr_h2.yml @@ -0,0 +1,43 @@ +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE file distributed with this work for additional +#information regarding copyright ownership. The ASF licenses this file to +#You under the Apache License, Version 2.0 (the "License"); you may not use +#this file except in compliance with the License. You may obtain a copy of +#the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +#by applicable law or agreed to in writing, software distributed under the +#License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +#OF ANY KIND, either express or implied. See the License for the specific +#language governing permissions and limitations under the License. + +name: 'table1' + +type: 'charset' + +vectors: + - 'VARCHAR_FIELD13' + - 'CHAR_FIELD16' + - 'CLOB_FIELD15' + +rowCount: '5' + +charSet: 'EUC-KR' + +create: 'CREATE TABLE table1 (int_field1 INT, varchar_field13 VARCHAR(256), clob_field15 CLOB, char_field16 CHAR(128));' + +data: + - 'INSERT INTO table1 VALUES (101,''char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.'', ''clob로 변환해야하는 char 인코딩을 가진 텍스트'', ''char 인코딩을 사용한 char 텍스트'');' + - 'INSERT INTO table1 VALUES (101,''char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.'', ''clob로 변환해야하는 char 인코딩을 가진 텍스트'', ''char 인코딩을 사용한 char 텍스트'');' + - 'INSERT INTO table1 VALUES (101,''char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.'', ''clob로 변환해야하는 char 인코딩을 가진 텍스트'', ''char 인코딩을 사용한 char 텍스트'');' + - 'INSERT INTO table1 VALUES (101,''char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.'', ''clob로 변환해야하는 char 인코딩을 가진 텍스트'', ''char 인코딩을 사용한 char 텍스트'');' + - 'INSERT INTO table1 VALUES (101,''char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.'', ''clob로 변환해야하는 char 인코딩을 가진 텍스트'', ''char 인코딩을 사용한 char 텍스트'');' + +query: 'select varchar_field13, clob_field15, char_field16 from table1' + +drop: 'DROP table table1;' + +values: + - 'VARCHAR_FIELD13=char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.,char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.,char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다., + char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.,char 인코딩을 사용하는 일부 텍스트를 varchar로 변환해야합니다.' + - 'CLOB_FIELD15=clob로 변환해야하는 char 인코딩을 가진 텍스트,clob로 변환해야하는 char 인코딩을 가진 텍스트,clob로 변환해야하는 char 인코딩을 가진 텍스트, + clob로 변환해야하는 char 인코딩을 가진 텍스트,clob로 변환해야하는 char 인코딩을 가진 텍스트' + - 'CHAR_FIELD16=char 인코딩을 사용한 char 텍스트,char 인코딩을 사용한 char 텍스트,char 인코딩을 사용한 char 텍스트,char 인코딩을 사용한 char 텍스트,char 인코딩을 사용한 char 텍스트' \ No newline at end of file From f685b932acbae78edd4e99b1ffd898bf704508f5 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 22 May 2018 07:42:48 -0400 Subject: [PATCH 114/129] Files committed Charset and Resultset based testcases --- .../jdbc/h2/JdbcToArrowCharSetTest.java | 160 ++++++++++ .../jdbc/h2/JdbcToArrowDataTypesTest.java | 192 ++++++------ .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 113 +++---- .../adapter/jdbc/h2/JdbcToArrowTest.java | 284 ++++++------------ .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 122 ++++---- 5 files changed, 435 insertions(+), 436 deletions(-) create mode 100644 java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java new file mode 100644 index 00000000000..2184d127f70 --- /dev/null +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.adapter.jdbc.h2; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; + +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; +import org.apache.arrow.adapter.jdbc.JdbcToArrow; +import org.apache.arrow.adapter.jdbc.Table; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getCharArrayWithCharSet; + +@RunWith(Parameterized.class) +public class JdbcToArrowCharSetTest extends AbstractJdbcToArrowTest { + private static final String VARCHAR = "VARCHAR_FIELD13"; + private static final String CHAR = "CHAR_FIELD16"; + private static final String CLOB = "CLOB_FIELD15"; + + private static final String[] testFiles = { + "h2/test1_charset_h2.yml", + "h2/test1_charset_ch_h2.yml", + "h2/test1_charset_jp_h2.yml", + "h2/test1_charset_kr_h2.yml" + }; + + /** + * Constructor which populate table object for each test iteration + * @param table + */ + public JdbcToArrowCharSetTest (Table table) { + this.table = table; + } + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest?characterEncoding=UTF-8"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowCharSetTest.class)); + } + + /** + * This method tests Chars, Varchars and Clob data with UTF-8 charset + * @throws SQLException + * @throws IOException + * @throws ClassNotFoundException + */ + @Test + public void testCharSetBasedData() throws SQLException, IOException, ClassNotFoundException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); + } + } + + /** + * This method tests Chars, Varchars and Clob data with UTF-8 charset using ResultSet + * @throws SQLException + * @throws IOException + * @throws ClassNotFoundException + */ + + @Test + public void testCharSetDataUsingResultSet() throws SQLException, IOException, ClassNotFoundException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + Calendar.getInstance())) { + testDataSets(root); + } + } + + /** + * This method tests Chars, Varchars and Clob data with UTF-8 charset using ResultSet and Allocator + * @throws SQLException + * @throws IOException + * @throws ClassNotFoundException + */ + + @Test + public void testCharSetDataUsingResultSetAndAllocator() throws SQLException, IOException, ClassNotFoundException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); + } + } + + /** + * This method calls the assert methods for various DataSets + * @param root + */ + public void testDataSets(VectorSchemaRoot root) throws UnsupportedEncodingException{ + assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), CLOB, StandardCharsets.UTF_8)); + + assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), VARCHAR, StandardCharsets.UTF_8)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), CHAR, StandardCharsets.UTF_8)); + } +} diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index 6d7feeef97b..303a772a1a2 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -17,9 +17,6 @@ */ package org.apache.arrow.adapter.jdbc.h2; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; @@ -38,16 +35,11 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; - import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -75,10 +67,8 @@ * */ @RunWith(Parameterized.class) -public class JdbcToArrowDataTypesTest { - private Connection conn = null; - private Table table; - +public class JdbcToArrowDataTypesTest extends AbstractJdbcToArrowTest { + private static final String BIGINT = "big_int"; private static final String BINARY = "binary"; private static final String BIT = "bit"; @@ -124,49 +114,6 @@ public class JdbcToArrowDataTypesTest { public JdbcToArrowDataTypesTest(Table table) { this.table = table; } - - /** - * This method creates Table object after reading YAML file - * @param ymlFilePath - * @return - * @throws IOException - */ - private static Table getTable(String ymlFilePath) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); - } - - - /** - * This method creates Connection object and DB table and also populate data into table for test - * @throws SQLException - * @throws ClassNotFoundException - */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } - } - - /** - * Clean up method to close connection after test completes - * @throws SQLException - */ - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } /** * This method returns collection of Table object for each test iteration @@ -177,12 +124,7 @@ public void destroy() throws SQLException { */ @Parameters public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; - int i = 0; - for (String testFile: testFiles) { - tableArr[i++] = new Object[]{getTable(testFile)}; - } - return Arrays.asList(tableArr); + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowDataTypesTest.class)); } /** @@ -194,52 +136,90 @@ public static Collection getTestData() throws SQLException, ClassNotFo public void testDBValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - switch (table.getType()) { - case BIGINT: - assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case BINARY:case BLOB: - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, table.getBinaryValues()); - break; - case BIT: - assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case BOOL: - assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getBoolValues()); - break; - case CHAR:case VARCHAR: case CLOB: - assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, table.getCharValues()); - break; - case DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case DECIMAL: - assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, table.getBigDecimalValues()); - break; - case DOUBLE: - assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, table.getDoubleValues()); - break; - case INT: - assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case SMALLINT: - assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case TINYINT: - assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case REAL: - assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, table.getFloatValues()); - break; - } + testDataSets(root); + } } - + + /** + * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet + * @throws SQLException + * @throws IOException + */ + @Test + public void testDBValuesUsingResultSet() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + Calendar.getInstance())) { + + testDataSets(root); + } + } + + /** + * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet And Allocator + * @throws SQLException + * @throws IOException + */ + @Test + public void testDBValuesUsingResultSetAndAllocator() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); + } + } + + /** + * This method calls the assert methods for various DataSets + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case BIGINT: + assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case BINARY:case BLOB: + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, table.getBinaryValues()); + break; + case BIT: + assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case BOOL: + assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getBoolValues()); + break; + case CHAR:case VARCHAR: case CLOB: + assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, table.getCharValues()); + break; + case DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case DECIMAL: + assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, table.getBigDecimalValues()); + break; + case DOUBLE: + assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, table.getDoubleValues()); + break; + case INT: + assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case SMALLINT: + assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case TINYINT: + assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case REAL: + assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, table.getFloatValues()); + break; + } + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index eee69ea1bc3..883f8192558 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -20,14 +20,13 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertNullValues; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; @@ -45,26 +44,18 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - /** * * JUnit Test class to test null values stored into Arrow vector for various datatypes for H2 database * */ @RunWith(Parameterized.class) -public class JdbcToArrowNullTest { - - private Connection conn = null; - private Table table; +public class JdbcToArrowNullTest extends AbstractJdbcToArrowTest { private static final String NULL = "null"; private static final String SELECTED_NULL_COLUMN = "selected_null_column"; @@ -83,85 +74,74 @@ public JdbcToArrowNullTest (Table table) { } /** - * This method creates Table object after reading YAML file - * @param ymlFilePath + * This method returns collection of Table object for each test iteration * @return - * @throws IOException - */ - private static Table getTable(String ymlFilePath) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } - - /** - * This method creates Connection object and DB table and also populate data into table for test * @throws SQLException * @throws ClassNotFoundException + * @throws IOException */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowNullTest.class)); } /** - * Clean up method to close connection after test completes + * This method tests null values stored into Arrow vector for various datatypes for H2 database * @throws SQLException + * @throws IOException */ - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } + @Test + public void testNullValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); + } } - + /** - * This method returns collection of Table object for each test iteration - * @return + * This method tests null values stored into Arrow vector for various datatypes for H2 database using ResultSet * @throws SQLException - * @throws ClassNotFoundException * @throws IOException */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; - int i = 0; - for (String testFile: testFiles) { - tableArr[i++] = new Object[]{getTable(testFile)}; - } - return Arrays.asList(tableArr); + @Test + public void testNullValuesUsingResultSet() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + Calendar.getInstance())) { + testDataSets(root); + } } /** - * This method tests null values stored into Arrow vector for various datatypes for H2 database + * This method tests null values stored into Arrow vector for various datatypes for H2 database using ResultSet And Allocator * @throws SQLException * @throws IOException */ @Test - public void testNullValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - switch (table.getType()) { - case NULL: - sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); - break; - case SELECTED_NULL_COLUMN: - sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount()); - break; - } + public void testNullValuesUsingResultSetAndAllocator() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); } } + /** + * This method calls the assert methods for various DataSets + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case NULL: + sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); + break; + case SELECTED_NULL_COLUMN: + sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount()); + break; + } + } /** * This method assert tests null values in vectors for all the datatypes * @param vectors @@ -209,4 +189,5 @@ public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSche assertNullValues((VarCharVector)root.getVector(vectors[11]), rowCount); assertNullValues((BitVector)root.getVector(vectors[12]), rowCount); } + } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index c6680af8921..a854145746f 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -18,8 +18,7 @@ package org.apache.arrow.adapter.jdbc.h2; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; @@ -37,17 +36,12 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; import java.io.IOException; -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -68,17 +62,20 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTinyIntVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarBinaryVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.hexStringToByteArray; - +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getLongValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getIntValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getBooleanValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getDecimalValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getDoubleValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getFloatValues; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getCharArray; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getBinaryValues; /** * */ @RunWith(Parameterized.class) -public class JdbcToArrowTest { - - private Connection conn = null; - private Table table; +public class JdbcToArrowTest extends AbstractJdbcToArrowTest { private static final String BIGINT = "BIGINT_FIELD5"; private static final String BINARY = "BINARY_FIELD12"; @@ -107,48 +104,6 @@ public class JdbcToArrowTest { public JdbcToArrowTest(Table table) { this.table = table; } - - /** - * This method creates Table object after reading YAML file - * @param ymlFilePath - * @return - * @throws IOException - */ - private static Table getTable(String ymlFilePath) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); - } - - /** - * This method creates Connection object and DB table and also populate data into table for test - * @throws SQLException - * @throws ClassNotFoundException - */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } - } - - /** - * Clean up method to close connection after test completes - * @throws SQLException - */ - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } /** * This method returns collection of Table object for each test iteration @@ -159,12 +114,7 @@ public void destroy() throws SQLException { */ @Parameters public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; - int i = 0; - for (String testFile: testFiles) { - tableArr[i++] = new Object[]{getTable(testFile)}; - } - return Arrays.asList(tableArr); + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTest.class)); } /** @@ -177,145 +127,93 @@ public void testDBValues() throws SQLException, IOException { try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), - getLongValues(table.getValues(), BIGINT)); - - assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), - getIntValues(table.getValues(), TINYINT)); - - assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), - getIntValues(table.getValues(), SMALLINT)); - - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), - getBinaryValues(table.getValues(), BINARY)); - - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), - getBinaryValues(table.getValues(), BLOB)); - - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), - getCharArrays(table.getValues(), CLOB)); - - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), - getCharArrays(table.getValues(), VARCHAR)); - - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), - getCharArrays(table.getValues(), CHAR)); - - assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), - getIntValues(table.getValues(), INT)); - - assertBitVectorValues((BitVector) root.getVector( BIT), table.getRowCount(), - getIntValues(table.getValues(), BIT)); - - assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), - getBooleanValues(table.getValues(), BOOL)); - - assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), - getLongValues(table.getValues(), DATE)); - - assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), - getLongValues(table.getValues(), TIME)); - - assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), - getLongValues(table.getValues(), TIMESTAMP)); - - assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), - getDecimalValues(table.getValues(), DECIMAL)); - - assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), - getDoubleValues(table.getValues(), DOUBLE)); - - assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), - getFloatValues(table.getValues(), REAL)); + testDataSets(root); } } - - private Integer [] getIntValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Integer [] valueArr = new Integer [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Integer.parseInt(data); - } - return valueArr; + /** + * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet + * @throws SQLException + * @throws IOException + */ + @Test + public void testDBValuesUsingResultSet() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + Calendar.getInstance())) { + testDataSets(root); + } } - private Boolean [] getBooleanValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Boolean [] valueArr = new Boolean [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().equals("1"); - } - return valueArr; + /** + * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet And Allocator + * @throws SQLException + * @throws IOException + */ + @Test + public void testDBValuesUsingResultSetAndAllocator() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { + + testDataSets(root); + } } - private BigDecimal [] getDecimalValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - BigDecimal [] valueArr = new BigDecimal [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr[i++] = new BigDecimal(data); - } - return valueArr; - } + /** + * This method calls the assert methods for various DataSets + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), + getLongValues(table.getValues(), BIGINT)); + + assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), + getIntValues(table.getValues(), TINYINT)); + + assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), + getIntValues(table.getValues(), SMALLINT)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), + getBinaryValues(table.getValues(), BINARY)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), + getBinaryValues(table.getValues(), BLOB)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + getCharArray(table.getValues(), CLOB)); - private Double [] getDoubleValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Double [] valueArr = new Double [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Double.parseDouble(data); - } - return valueArr; - } - private Float [] getFloatValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Float [] valueArr = new Float [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Float.parseFloat(data); - } - return valueArr; - } - private Long [] getLongValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Long [] valueArr = new Long [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Long.parseLong(data); - } - return valueArr; + assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + getCharArray(table.getValues(), VARCHAR)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + getCharArray(table.getValues(), CHAR)); + + assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), + getIntValues(table.getValues(), INT)); + + assertBitVectorValues((BitVector) root.getVector( BIT), table.getRowCount(), + getIntValues(table.getValues(), BIT)); + + assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), + getBooleanValues(table.getValues(), BOOL)); + + assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), + getLongValues(table.getValues(), DATE)); + + assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), + getLongValues(table.getValues(), TIME)); + + assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), + getLongValues(table.getValues(), TIMESTAMP)); + + assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), + getDecimalValues(table.getValues(), DECIMAL)); + + assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), + getDoubleValues(table.getValues(), DOUBLE)); + + assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), + getFloatValues(table.getValues(), REAL)); } - private byte [][] getCharArrays(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().getBytes(); - } - return valueArr; - } - - private byte [][] getBinaryValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = hexStringToByteArray(data.trim()); - } - return valueArr; - } - - private String [] getValues(String[] values, String dataType) { - String value = ""; - for(String val : values) { - if(val.startsWith(dataType)) { - value = val.split("=")[1]; - break; - } - } - return value.split(","); - } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 595ddabf04b..59700e30ab2 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -17,21 +17,18 @@ */ package org.apache.arrow.adapter.jdbc.h2; - import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeStampVectorValues; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertTimeVectorValues; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.TimeZone; - +import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; @@ -39,26 +36,18 @@ import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - /** * * JUnit Test class to test TimeZone based Date, Time and Timestamp datatypes for H2 database * */ @RunWith(Parameterized.class) -public class JdbcToArrowTimeZoneTest { - - private Connection conn = null; - private Table table; +public class JdbcToArrowTimeZoneTest extends AbstractJdbcToArrowTest { private static final String EST_DATE = "est_date"; private static final String EST_TIME = "est_time"; @@ -91,86 +80,77 @@ public JdbcToArrowTimeZoneTest (Table table) { } /** - * This method creates Table object after reading YAML file - * @param ymlFilePath + * This method returns collection of Table object for each test iteration * @return - * @throws IOException - */ - private static Table getTable(String ymlFilePath) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - JdbcToArrowDataTypesTest.class.getClassLoader().getResourceAsStream(ymlFilePath), - Table.class); - } - - /** - * This method creates Connection object and DB table and also populate data into table for test * @throws SQLException * @throws ClassNotFoundException + * @throws IOException */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTimeZoneTest.class)); } /** - * Clean up method to close connection after test completes + * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database * @throws SQLException + * @throws IOException */ - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } - } + @Test + public void testTimeZoneBasedValues() throws SQLException, IOException { + try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { + + testDataSets(root); + } + } /** - * This method returns collection of Table object for each test iteration - * @return + * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database using ResultSet * @throws SQLException - * @throws ClassNotFoundException * @throws IOException */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; - int i = 0; - for (String testFile: testFiles) { - tableArr[i++] = new Object[]{getTable(testFile)}; - } - return Arrays.asList(tableArr); + @Test + public void testTimeZoneBasedValuesUsingResultSet() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { + + testDataSets(root); + } } /** - * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database + * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database using ResultSet and Allocator * @throws SQLException * @throws IOException */ @Test - public void testTimeZoneBasedValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { - switch (table.getType()) { - case EST_DATE: case GMT_DATE: case PST_DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case EST_TIME: case GMT_TIME: case PST_TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case EST_TIMESTAMP: case GMT_TIMESTAMP: case PST_TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - } + public void testTimeZoneBasedValuesUsingResultSetAndAllocator() throws SQLException, IOException { + try (Statement stmt = conn.createStatement(); + VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { + + testDataSets(root); } } + /** + * This method calls the assert methods for various DataSets + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case EST_DATE: case GMT_DATE: case PST_DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIME: case GMT_TIME: case PST_TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIMESTAMP: case GMT_TIMESTAMP: case PST_TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + } + } + } From dcaa74a0d82493720c18ac93e0da2da95eed3435 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 22 May 2018 07:43:33 -0400 Subject: [PATCH 115/129] Files committed Charset and Resultset based testcases --- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 73 ++++++++++++++++++- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 099f304abb3..eeea07ec477 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -18,16 +18,25 @@ package org.apache.arrow.adapter.jdbc; +import java.io.IOException; import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.sql.Statement; +import org.junit.After; +import org.junit.Before; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + /** * Class to abstract out some common test functionality for testing JDBC to Arrow. */ public abstract class AbstractJdbcToArrowTest { - + protected Connection conn = null; + protected Table table; protected void createTestData(Connection conn, Table table) throws Exception { - Statement stmt = null; try { //create the table and insert the data and once done drop the table @@ -48,7 +57,6 @@ protected void createTestData(Connection conn, Table table) throws Exception { } - protected void deleteTestData(Connection conn, Table table) throws Exception { Statement stmt = null; try { @@ -63,4 +71,63 @@ protected void deleteTestData(Connection conn, Table table) throws Exception { } } } + + /** + * This method creates Table object after reading YAML file + * @param ymlFilePath + * @return + * @throws IOException + */ + protected static Table getTable(String ymlFilePath, Class clss) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( + clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + } + + + /** + * This method creates Connection object and DB table and also populate data into table for test + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } + } + } + + /** + * Clean up method to close connection after test completes + * @throws SQLException + */ + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; + } + } + + /** + * This method returns collection of Table object for each test iteration + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + public static Object[][] prepareTestData(String[] testFiles, Class clss) throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile: testFiles) { + tableArr[i++] = new Object[]{getTable(testFile, clss)}; + } + return tableArr; + } } From 239fca65590a07936d1ce787753cba969568265e Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 22 May 2018 07:43:48 -0400 Subject: [PATCH 116/129] Files committed Charset and Resultset based testcases --- .../adapter/jdbc/JdbcToArrowTestHelper.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 8eaaa703da2..6abfdff35fd 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -18,7 +18,9 @@ package org.apache.arrow.adapter.jdbc; +import java.io.UnsupportedEncodingException; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.util.Arrays; import org.apache.arrow.vector.BaseValueVector; @@ -35,6 +37,8 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.hexStringToByteArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; @@ -174,5 +178,105 @@ public static byte[] hexStringToByteArray(String s) { } return data; } + + public static Integer [] getIntValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Integer [] valueArr = new Integer [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Integer.parseInt(data); + } + return valueArr; + } + + public static Boolean [] getBooleanValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Boolean [] valueArr = new Boolean [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().equals("1"); + } + return valueArr; + } + + public static BigDecimal [] getDecimalValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + BigDecimal [] valueArr = new BigDecimal [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr[i++] = new BigDecimal(data); + } + return valueArr; + } + public static Double [] getDoubleValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Double [] valueArr = new Double [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Double.parseDouble(data); + } + return valueArr; + } + + public static Float [] getFloatValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Float [] valueArr = new Float [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Float.parseFloat(data); + } + return valueArr; + } + + public static Long [] getLongValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + Long [] valueArr = new Long [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Long.parseLong(data); + } + return valueArr; + } + + public static byte [][] getCharArray(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().getBytes(); + } + return valueArr; + } + + public static byte [][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) throws UnsupportedEncodingException { + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().getBytes(charSet); + } + return valueArr; + } + + public static byte [][] getBinaryValues(String[] values, String dataType) { + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = hexStringToByteArray(data.trim()); + } + return valueArr; + } + + public static String [] getValues(String[] values, String dataType) { + String value = ""; + for(String val : values) { + if(val.startsWith(dataType)) { + value = val.split("=")[1]; + break; + } + } + return value.split(","); + } } From 13ec352739d0f9e2f4d8d053a7b6d443e7d48d6f Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 22 May 2018 07:45:08 -0400 Subject: [PATCH 117/129] File committed for resolving exception related to closing RootAllocator --- .../main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 0bdcc007e3e..617cbd15339 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -153,7 +153,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); - rootAllocator.close(); + return root; } From 67593cdddcb69a1062f6cbef7f22789cfede488a Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:05:42 -0400 Subject: [PATCH 118/129] File committed for Code Coverage related changes --- java/adapter/jdbc/pom.xml | 66 ++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 5a879247746..d8dd422ec68 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -29,6 +29,7 @@ arrow-memory ${project.version} + org.apache.arrow @@ -41,7 +42,6 @@ ${dep.guava.version} - junit @@ -49,6 +49,7 @@ ${dep.junit.version} test + com.h2database @@ -83,18 +84,55 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - - - UTC - - + + + + org.jacoco + jacoco-maven-plugin + 0.8.1 + + + + default-prepare-agent + + prepare-agent + + + + ${project.build.directory}/coverage-reports/jacoco-ut.exec + + surefireArgLine + + + + + post-unit-test + test + + report + + + + ${project.build.directory}/coverage-reports/jacoco-ut.exec + + true + ${project.build.directory}/jacoco-ut + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + UTC + + + ${surefireArgLine} + + + + - - - From d260342c314ee919458cbca20a7c9faa4c0098a1 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:06:39 -0400 Subject: [PATCH 119/129] File committed for Code Coverage related changes --- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 54 ++++++------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index eeea07ec477..2c76c178ea0 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -23,10 +23,10 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; - +import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; - +import org.junit.Test; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -36,41 +36,6 @@ public abstract class AbstractJdbcToArrowTest { protected Connection conn = null; protected Table table; - protected void createTestData(Connection conn, Table table) throws Exception { - Statement stmt = null; - try { - //create the table and insert the data and once done drop the table - stmt = conn.createStatement(); - stmt.executeUpdate(table.getCreate()); - - for (String insert: table.getData()) { - stmt.executeUpdate(insert); - } - - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - stmt.close(); - } - } - - } - - protected void deleteTestData(Connection conn, Table table) throws Exception { - Statement stmt = null; - try { - stmt = conn.createStatement(); - stmt.executeUpdate(table.getDrop()); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - stmt.close(); - } - } - } /** * This method creates Table object after reading YAML file @@ -130,4 +95,19 @@ public static Object[][] prepareTestData(String[] testFiles, Class clss) throws } return tableArr; } + + /** + * Abstract method to implement test Functionality to test JdbcToArrow methods + * @throws SQLException + * @throws IOException + */ + @Test + public abstract void testJdbcToArroValues() throws SQLException, IOException; + + /** + * Abstract method to implement logic to assert test various datatype values + * @param root + */ + public abstract void testDataSets(VectorSchemaRoot root); + } From 6254260753355ba6e19c72411b3e682b52dc0b8a Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:07:33 -0400 Subject: [PATCH 120/129] File committed for Code Coverage related changes --- .../org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 6abfdff35fd..37580f0f2ea 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -18,7 +18,6 @@ package org.apache.arrow.adapter.jdbc; -import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.nio.charset.Charset; import java.util.Arrays; @@ -38,7 +37,6 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; -import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.hexStringToByteArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; @@ -249,7 +247,7 @@ public static byte[] hexStringToByteArray(String s) { return valueArr; } - public static byte [][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) throws UnsupportedEncodingException { + public static byte [][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) { String[] dataArr= getValues(values, dataType); byte [][] valueArr = new byte [dataArr.length][]; int i =0; From c9b22fee1a5f5c818ea97e5e0d481518c7709228 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:08:03 -0400 Subject: [PATCH 121/129] File committed for Code Coverage related changes --- .../jdbc/h2/JdbcToArrowCharSetTest.java | 62 +++++-------------- 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java index 2184d127f70..db3620f38c7 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -20,7 +20,6 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.sql.DriverManager; import java.sql.SQLException; @@ -43,6 +42,12 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getCharArrayWithCharSet; +/** + * + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with UTF-8 Charset, including + * the multi-byte CJK characters for H2 database + * + */ @RunWith(Parameterized.class) public class JdbcToArrowCharSetTest extends AbstractJdbcToArrowTest { private static final String VARCHAR = "VARCHAR_FIELD13"; @@ -94,60 +99,27 @@ public void setUp() throws SQLException, ClassNotFoundException { public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowCharSetTest.class)); } - - /** - * This method tests Chars, Varchars and Clob data with UTF-8 charset - * @throws SQLException - * @throws IOException - * @throws ClassNotFoundException - */ - @Test - public void testCharSetBasedData() throws SQLException, IOException, ClassNotFoundException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } - } - - /** - * This method tests Chars, Varchars and Clob data with UTF-8 charset using ResultSet - * @throws SQLException - * @throws IOException - * @throws ClassNotFoundException - */ - - @Test - public void testCharSetDataUsingResultSet() throws SQLException, IOException, ClassNotFoundException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - Calendar.getInstance())) { - testDataSets(root); - } - } /** - * This method tests Chars, Varchars and Clob data with UTF-8 charset using ResultSet and Allocator - * @throws SQLException - * @throws IOException - * @throws ClassNotFoundException + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with UTF-8 Charset, including + * the multi-byte CJK characters */ - @Test - public void testCharSetDataUsingResultSetAndAllocator() throws SQLException, IOException, ClassNotFoundException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); } /** * This method calls the assert methods for various DataSets * @param root */ - public void testDataSets(VectorSchemaRoot root) throws UnsupportedEncodingException{ + public void testDataSets(VectorSchemaRoot root) { assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), getCharArrayWithCharSet(table.getValues(), CLOB, StandardCharsets.UTF_8)); From fa65a310af6740e66ed3a61ea72192d6b58d5aa7 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:08:23 -0400 Subject: [PATCH 122/129] File committed for Code Coverage related changes --- .../jdbc/h2/JdbcToArrowDataTypesTest.java | 52 ++++--------------- 1 file changed, 11 insertions(+), 41 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index 303a772a1a2..c615d0b77d3 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -41,7 +41,6 @@ import org.junit.runners.Parameterized.Parameters; import java.io.IOException; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; @@ -63,7 +62,8 @@ /** * - * JUnit Test class to test various datatypes converted into Arrow vector for H2 database + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database + * using multiple test data files * */ @RunWith(Parameterized.class) @@ -128,47 +128,17 @@ public static Collection getTestData() throws SQLException, ClassNotFo } /** - * This method tests various datatypes converted into Arrow vector for H2 database - * @throws SQLException - * @throws IOException + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes */ @Test - public void testDBValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - testDataSets(root); - - } - } - - /** - * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet - * @throws SQLException - * @throws IOException - */ - @Test - public void testDBValuesUsingResultSet() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - Calendar.getInstance())) { - - testDataSets(root); - } - } - - /** - * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet And Allocator - * @throws SQLException - * @throws IOException - */ - @Test - public void testDBValuesUsingResultSetAndAllocator() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(),new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(),new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); } /** From 9a2f463cba0ad15b14dd4ec27d3c1b6689295ebc Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:08:40 -0400 Subject: [PATCH 123/129] File committed for Code Coverage related changes --- .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 51 ++++--------------- 1 file changed, 11 insertions(+), 40 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 883f8192558..4be27d9b46d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; @@ -51,7 +50,7 @@ /** * - * JUnit Test class to test null values stored into Arrow vector for various datatypes for H2 database + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with null values for H2 database * */ @RunWith(Parameterized.class) @@ -86,48 +85,20 @@ public static Collection getTestData() throws SQLException, ClassNotFo } /** - * This method tests null values stored into Arrow vector for various datatypes for H2 database - * @throws SQLException - * @throws IOException - */ - @Test - public void testNullValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } - } - - /** - * This method tests null values stored into Arrow vector for various datatypes for H2 database using ResultSet - * @throws SQLException - * @throws IOException + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null values */ @Test - public void testNullValuesUsingResultSet() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - Calendar.getInstance())) { - testDataSets(root); - } - } - - /** - * This method tests null values stored into Arrow vector for various datatypes for H2 database using ResultSet And Allocator - * @throws SQLException - * @throws IOException - */ - @Test - public void testNullValuesUsingResultSetAndAllocator() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow (conn,table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow (conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); } + /** * This method calls the assert methods for various DataSets * @param root From 7125d6e8d5e8069816cef7eb08401569656cf8a3 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:08:57 -0400 Subject: [PATCH 124/129] File committed for Code Coverage related changes --- .../adapter/jdbc/h2/JdbcToArrowTest.java | 55 +++++-------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index a854145746f..8e0606985ac 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -43,7 +43,6 @@ import java.io.IOException; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; @@ -72,6 +71,9 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getBinaryValues; /** + * + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database + * using single test data file * */ @RunWith(Parameterized.class) @@ -116,47 +118,20 @@ public JdbcToArrowTest(Table table) { public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTest.class)); } - - /** - * This method tests various datatypes converted into Arrow vector for H2 database - * @throws SQLException - * @throws IOException - */ - @Test - public void testDBValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } - } - /** - * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet - * @throws SQLException - * @throws IOException - */ - @Test - public void testDBValuesUsingResultSet() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - Calendar.getInstance())) { - testDataSets(root); - } - } - + /** - * This method tests various datatypes converted into Arrow vector for H2 database using ResultSet And Allocator - * @throws SQLException - * @throws IOException + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one test data file + * */ @Test - public void testDBValuesUsingResultSetAndAllocator() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())) { - - testDataSets(root); - } + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); } /** @@ -215,5 +190,5 @@ public void testDataSets(VectorSchemaRoot root) { assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), getFloatValues(table.getValues(), REAL)); } - + } From b124ece8cc471dd7a38f5640d4e85beb871ac002 Mon Sep 17 00:00:00 2001 From: yashpal Date: Tue, 29 May 2018 05:09:13 -0400 Subject: [PATCH 125/129] File committed for Code Coverage related changes --- .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 55 +++++-------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 59700e30ab2..438a0228e7d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; @@ -43,9 +42,11 @@ /** * - * JUnit Test class to test TimeZone based Date, Time and Timestamp datatypes for H2 database + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with TimeZone based Date, + * Time and Timestamp datatypes for H2 database * */ + @RunWith(Parameterized.class) public class JdbcToArrowTimeZoneTest extends AbstractJdbcToArrowTest { @@ -92,47 +93,17 @@ public static Collection getTestData() throws SQLException, ClassNotFo } /** - * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database - * @throws SQLException - * @throws IOException - */ - @Test - public void testTimeZoneBasedValues() throws SQLException, IOException { - try (VectorSchemaRoot root = JdbcToArrow.sqlToArrow(conn, table.getQuery(), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { - - testDataSets(root); - } - } - - /** - * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database using ResultSet - * @throws SQLException - * @throws IOException - */ - @Test - public void testTimeZoneBasedValuesUsingResultSet() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { - - testDataSets(root); - } - } - - /** - * This method tests TimeZone based Date, Time and Timestamp datatypes for H2 database using ResultSet and Allocator - * @throws SQLException - * @throws IOException - */ + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with TimeZone based Date, + * Time and Timestamp datatype + */ @Test - public void testTimeZoneBasedValuesUsingResultSetAndAllocator() throws SQLException, IOException { - try (Statement stmt = conn.createStatement(); - VectorSchemaRoot root = JdbcToArrow.sqlToArrow(stmt.executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))) { - - testDataSets(root); - } + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); } /** From e189b143c236a414881ef4fbdf036bb47c4a636e Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 31 May 2018 05:22:24 -0400 Subject: [PATCH 126/129] File committed for the changes made as part of review comment implementation --- java/adapter/jdbc/pom.xml | 92 +++++++++++++-------------------------- 1 file changed, 30 insertions(+), 62 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index d8dd422ec68..17d2381213a 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -1,17 +1,18 @@ + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> 4.0.0 + org.apache.arrow arrow-java-root @@ -21,6 +22,7 @@ arrow-jdbc Arrow JDBC Adapter http://maven.apache.org + @@ -36,6 +38,7 @@ arrow-vector ${project.version} + com.google.guava guava @@ -57,82 +60,47 @@ 1.4.196 test + com.fasterxml.jackson.dataformat jackson-dataformat-yaml ${dep.jackson.version} test + com.fasterxml.jackson.core jackson-databind ${dep.jackson.version} test + com.fasterxml.jackson.core jackson-core ${dep.jackson.version} test + com.fasterxml.jackson.core jackson-annotations ${dep.jackson.version} test - - - - - org.jacoco - jacoco-maven-plugin - 0.8.1 - - - - default-prepare-agent - - prepare-agent - - - - ${project.build.directory}/coverage-reports/jacoco-ut.exec - - surefireArgLine - - - - - post-unit-test - test - - report - - - - ${project.build.directory}/coverage-reports/jacoco-ut.exec - - true - ${project.build.directory}/jacoco-ut - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - UTC - - - ${surefireArgLine} - - - - - - + + + + org.apache.maven.plugins + maven-surefire-plugin + + + UTC + + + + + + \ No newline at end of file From c66f4a2d91202bfc1b34a6db99dbc0433a890e67 Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 31 May 2018 05:22:46 -0400 Subject: [PATCH 127/129] File committed for the changes made as part of review comment implementation --- .../java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 8 +++++--- .../org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 617cbd15339..5a7e1c77b11 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -30,6 +30,8 @@ import java.sql.Statement; import java.sql.ResultSet; import java.util.Calendar; +import java.util.Locale; +import java.util.TimeZone; /** * Utility class to convert JDBC objects to columnar Arrow format objects. @@ -85,7 +87,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); - return sqlToArrow(connection, query, allocator, Calendar.getInstance()); + return sqlToArrow(connection, query, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); } /** @@ -121,7 +123,7 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, B public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - return sqlToArrow(resultSet, Calendar.getInstance()); + return sqlToArrow(resultSet, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); } /** @@ -136,7 +138,7 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator all Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); - return sqlToArrow(resultSet, allocator, Calendar.getInstance()); + return sqlToArrow(resultSet, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); } /** diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index a96e45e6bd0..e48255e313a 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -304,7 +304,6 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calen case Types.VARBINARY: case Types.LONGVARBINARY: updateVector((VarBinaryVector)root.getVector(columnName), -// rs.getBytes(i), !rs.wasNull(), rowCount); rs.getBinaryStream(i), !rs.wasNull(), rowCount); break; case Types.ARRAY: @@ -474,7 +473,7 @@ private static void updateVector(VarBinaryVector varBinaryVector, InputStream is if (read == -1) { break; } - arrowBuf.setBytes(total, new ByteArrayInputStream(bytes, 0, read), read); + arrowBuf.setBytes(total, bytes, total, read); total += read; } holder.end = total; From abaea4ed3450594b8a4b69e1468d6f1aeb9b2ee6 Mon Sep 17 00:00:00 2001 From: yashpal Date: Thu, 31 May 2018 05:23:06 -0400 Subject: [PATCH 128/129] File committed for the changes made as part of review comment implementation --- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 12 +- .../adapter/jdbc/JdbcToArrowTestHelper.java | 181 +++++++++--------- .../org/apache/arrow/adapter/jdbc/Table.java | 56 +++--- 3 files changed, 125 insertions(+), 124 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 2c76c178ea0..19438c270dd 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -36,7 +36,7 @@ public abstract class AbstractJdbcToArrowTest { protected Connection conn = null; protected Table table; - + /** * This method creates Table object after reading YAML file * @param ymlFilePath @@ -45,7 +45,7 @@ public abstract class AbstractJdbcToArrowTest { */ protected static Table getTable(String ymlFilePath, Class clss) throws IOException { return new ObjectMapper(new YAMLFactory()).readValue( - clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); } @@ -67,7 +67,7 @@ public void setUp() throws SQLException, ClassNotFoundException { } } } - + /** * Clean up method to close connection after test completes * @throws SQLException @@ -79,7 +79,7 @@ public void destroy() throws SQLException { conn = null; } } - + /** * This method returns collection of Table object for each test iteration * @return @@ -88,14 +88,14 @@ public void destroy() throws SQLException { * @throws IOException */ public static Object[][] prepareTestData(String[] testFiles, Class clss) throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; + Object[][] tableArr = new Object[testFiles.length][]; int i = 0; for (String testFile: testFiles) { tableArr[i++] = new Object[]{getTable(testFile, clss)}; } return tableArr; } - + /** * Abstract method to implement test Functionality to test JdbcToArrow methods * @throws SQLException diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 37580f0f2ea..8fabc4dcdae 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -20,8 +20,6 @@ import java.math.BigDecimal; import java.nio.charset.Charset; -import java.util.Arrays; - import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -36,10 +34,11 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertArrayEquals; + /** * This is a Helper class which has functionalities to read and assert the values from the given FieldVector object * @@ -50,13 +49,13 @@ public static void assertIntVectorValues(IntVector intVector, int rowCount, Inte assertEquals(rowCount, intVector.getValueCount()); for(int j = 0; j < intVector.getValueCount(); j++) { - assertEquals(values[j].intValue(), intVector.get(j)); + assertEquals(values[j].intValue(), intVector.get(j)); } } public static void assertBooleanVectorValues(BitVector bitVector, int rowCount, Boolean[] values){ assertEquals(rowCount, bitVector.getValueCount()); - + for(int j = 0; j < bitVector.getValueCount(); j++){ assertEquals(values[j].booleanValue(), bitVector.get(j) == 1); } @@ -98,7 +97,7 @@ public static void assertDecimalVectorValues(DecimalVector decimalVector, int ro assertEquals(rowCount, decimalVector.getValueCount()); for(int j = 0; j < decimalVector.getValueCount(); j++){ - assertNotNull(decimalVector.getObject(j)); + assertNotNull(decimalVector.getObject(j)); assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); } } @@ -123,7 +122,7 @@ public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int r assertEquals(rowCount, timeMilliVector.getValueCount()); for(int j = 0; j < timeMilliVector.getValueCount(); j++){ - assertEquals(values[j].longValue(), timeMilliVector.get(j)); + assertEquals(values[j].longValue(), timeMilliVector.get(j)); } } @@ -147,7 +146,7 @@ public static void assertVarBinaryVectorValues (VarBinaryVector varBinaryVector, assertEquals(rowCount, varBinaryVector.getValueCount()); for(int j = 0; j < varBinaryVector.getValueCount(); j++){ - assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varBinaryVector.get(j))); + assertArrayEquals(values[j], varBinaryVector.get(j)); } } @@ -155,15 +154,15 @@ public static void assertVarcharVectorValues(VarCharVector varCharVector, int ro assertEquals(rowCount, varCharVector.getValueCount()); for(int j = 0; j < varCharVector.getValueCount(); j++){ - assertEquals(Arrays.hashCode(values[j]), Arrays.hashCode(varCharVector.get(j))); + assertArrayEquals(values[j], varCharVector.get(j)); } } - + public static void assertNullValues(BaseValueVector vector, int rowCount) { assertEquals(rowCount, vector.getValueCount()); for(int j = 0; j < vector.getValueCount(); j++) { - assertTrue(vector.isNull(j)); + assertTrue(vector.isNull(j)); } } @@ -172,109 +171,109 @@ public static byte[] hexStringToByteArray(String s) { byte[] data = new byte[len / 2]; for (int i = 0; i < len; i += 2) { data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) - + Character.digit(s.charAt(i+1), 16)); + + Character.digit(s.charAt(i+1), 16)); } return data; } - + public static Integer [] getIntValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Integer [] valueArr = new Integer [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Integer.parseInt(data); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + Integer [] valueArr = new Integer [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Integer.parseInt(data); + } + return valueArr; } - + public static Boolean [] getBooleanValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Boolean [] valueArr = new Boolean [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().equals("1"); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + Boolean [] valueArr = new Boolean [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().equals("1"); + } + return valueArr; } public static BigDecimal [] getDecimalValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - BigDecimal [] valueArr = new BigDecimal [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr[i++] = new BigDecimal(data); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + BigDecimal [] valueArr = new BigDecimal [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr[i++] = new BigDecimal(data); + } + return valueArr; } public static Double [] getDoubleValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Double [] valueArr = new Double [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Double.parseDouble(data); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + Double [] valueArr = new Double [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Double.parseDouble(data); + } + return valueArr; } - + public static Float [] getFloatValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Float [] valueArr = new Float [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Float.parseFloat(data); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + Float [] valueArr = new Float [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Float.parseFloat(data); + } + return valueArr; } - + public static Long [] getLongValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Long [] valueArr = new Long [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Long.parseLong(data); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + Long [] valueArr = new Long [dataArr.length]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = Long.parseLong(data); + } + return valueArr; } - + public static byte [][] getCharArray(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().getBytes(); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().getBytes(); + } + return valueArr; } - + public static byte [][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().getBytes(charSet); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = data.trim().getBytes(charSet); + } + return valueArr; } - + public static byte [][] getBinaryValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = hexStringToByteArray(data.trim()); - } - return valueArr; + String[] dataArr= getValues(values, dataType); + byte [][] valueArr = new byte [dataArr.length][]; + int i =0; + for(String data : dataArr) { + valueArr [i++] = hexStringToByteArray(data.trim()); + } + return valueArr; } - + public static String [] getValues(String[] values, String dataType) { - String value = ""; - for(String val : values) { - if(val.startsWith(dataType)) { - value = val.split("=")[1]; - break; - } - } - return value.split(","); + String value = ""; + for(String val : values) { + if(val.startsWith(dataType)) { + value = val.split("=")[1]; + break; + } + } + return value.split(","); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 3c99ceb1578..b4305eb38b9 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -28,7 +28,6 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public class Table { - private String name; private String type; private String vector; @@ -93,6 +92,7 @@ public Integer[] getIntValues() { } return arr; } + public Boolean[] getBoolValues() { Boolean[] arr = new Boolean[values.length]; int i = 0; @@ -101,6 +101,7 @@ public Boolean[] getBoolValues() { } return arr; } + public BigDecimal[] getBigDecimalValues() { BigDecimal[] arr = new BigDecimal[values.length]; int i = 0; @@ -109,6 +110,7 @@ public BigDecimal[] getBigDecimalValues() { } return arr; } + public Double[] getDoubleValues() { Double[] arr = new Double[values.length]; int i = 0; @@ -180,30 +182,30 @@ public void setDrop(String drop) { } public String getTimezone() { - return timezone; - } - - public void setTimezone(String timezone) { - this.timezone = timezone; - } - - public String[] getVectors() { - return vectors; - } - - public void setVectors(String[] vectors) { - this.vectors = vectors; - } - - public int getRowCount() { - return rowCount; - } - - public void setRowCount(int rowCount) { - this.rowCount = rowCount; - } - - private byte[][] getByteArray(String[] data) { + return timezone; + } + + public void setTimezone(String timezone) { + this.timezone = timezone; + } + + public String[] getVectors() { + return vectors; + } + + public void setVectors(String[] vectors) { + this.vectors = vectors; + } + + public int getRowCount() { + return rowCount; + } + + public void setRowCount(int rowCount) { + this.rowCount = rowCount; + } + + private byte[][] getByteArray(String[] data) { byte[][] byteArr = new byte[data.length][]; for (int i = 0; i < data.length; i++) { @@ -226,8 +228,8 @@ private static byte[] hexStringToByteArray(String s) { byte[] data = new byte[len / 2]; for (int i = 0; i < len; i += 2) { data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) - + Character.digit(s.charAt(i + 1), 16)); + + Character.digit(s.charAt(i + 1), 16)); } return data; } -} +} \ No newline at end of file From dd1ffa4d70835f3bdcf3c67288739c7cf5b8ef8e Mon Sep 17 00:00:00 2001 From: Atul Dambalkar Date: Sat, 16 Jun 2018 06:01:52 -0700 Subject: [PATCH 129/129] Fixed indentation for the code as per checkstyle and rest of the Arrow code. --- .../arrow/adapter/jdbc/JdbcToArrow.java | 232 ++--- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 814 +++++++++--------- .../adapter/jdbc/AbstractJdbcToArrowTest.java | 143 +-- .../adapter/jdbc/JdbcToArrowTestHelper.java | 361 ++++---- .../org/apache/arrow/adapter/jdbc/Table.java | 382 ++++---- .../jdbc/h2/JdbcToArrowCharSetTest.java | 158 ++-- .../jdbc/h2/JdbcToArrowDataTypesTest.java | 239 ++--- .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 208 ++--- .../adapter/jdbc/h2/JdbcToArrowTest.java | 220 ++--- .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 154 ++-- 10 files changed, 1469 insertions(+), 1442 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 5a7e1c77b11..436a570b14d 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,22 +22,22 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import com.google.common.base.Preconditions; - import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; -import java.sql.Statement; import java.sql.ResultSet; +import java.sql.Statement; import java.util.Calendar; import java.util.Locale; import java.util.TimeZone; +import com.google.common.base.Preconditions; + /** * Utility class to convert JDBC objects to columnar Arrow format objects. - * + *

* This utility uses following data mapping to map JDBC/SQL datatype to Arrow data types. - * + *

* CHAR --> ArrowType.Utf8 * NCHAR --> ArrowType.Utf8 * VARCHAR --> ArrowType.Utf8 @@ -62,120 +62,120 @@ * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) * CLOB --> ArrowType.Utf8 * BLOB --> ArrowType.Binary - * + *

* TODO: At this time, SQL Data type java.sql.Types.ARRAY is still not supported. * * @since 0.10.0 */ public class JdbcToArrow { - /** - * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. - * This method uses the default Calendar instance with default TimeZone and Locale as returned by the JVM. - * If you wish to use specific TimeZone or Locale for any Date, Time and Timestamp datasets, you may want use - * overloaded API that taken Calendar object instance. - * - * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed - * the connection object it's the responsibility of the caller to close or return the connection to the pool. - * @param query The DB Query to fetch the data. - * @param allocator Memory allocator - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. - */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException, IOException { - Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); - Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); - Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); - - return sqlToArrow(connection, query, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); - } - - /** - * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. - * - * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed - * the connection object it's the responsibility of the caller to close or return the connection to the pool. - * @param query The DB Query to fetch the data. - * @param allocator Memory allocator - * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. - */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { - Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); - Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); - Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); - Preconditions.checkNotNull(calendar, "Calendar object can not be null"); - - try (Statement stmt = connection.createStatement()) { - return sqlToArrow(stmt.executeQuery(query), allocator, calendar); - } - } - - /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. This method - * uses the default RootAllocator and Calendar object. - * - * @param resultSet - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException - */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { - Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - - return sqlToArrow(resultSet, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); - } - - /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. - * - * @param resultSet - * @param allocator Memory allocator - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException - */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException, IOException { - Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); - - return sqlToArrow(resultSet, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); - } - - /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. - * - * @param resultSet - * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException - */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException, IOException { - Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(calendar, "Calendar object can not be null"); - - RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); - VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); - - return root; - } - - /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. - * - * @param resultSet - * @param allocator Memory allocator to use. - * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. - * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException - */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { - Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); - Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); - Preconditions.checkNotNull(calendar, "Calendar object can not be null"); - - VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), calendar), allocator); - JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, calendar); - return root; + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * This method uses the default Calendar instance with default TimeZone and Locale as returned by the JVM. + * If you wish to use specific TimeZone or Locale for any Date, Time and Timestamp datasets, you may want use + * overloaded API that taken Calendar object instance. + * + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. + */ + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator) throws SQLException, IOException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + + return sqlToArrow(connection, query, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + } + + /** + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * + * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed + * the connection object it's the responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statement objects. + */ + public static VectorSchemaRoot sqlToArrow(Connection connection, String query, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { + Preconditions.checkNotNull(connection, "JDBC connection object can not be null"); + Preconditions.checkArgument(query != null && query.length() > 0, "SQL query can not be null or empty"); + Preconditions.checkNotNull(allocator, "Memory allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + + try (Statement stmt = connection.createStatement()) { + return sqlToArrow(stmt.executeQuery(query), allocator, calendar); } + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. This method + * uses the default RootAllocator and Calendar object. + * + * @param resultSet + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLException, IOException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + + return sqlToArrow(resultSet, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param allocator Memory allocator + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator) throws SQLException, IOException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + + return sqlToArrow(resultSet, allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT)); + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException, IOException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + + RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot root = sqlToArrow(resultSet, rootAllocator, calendar); + + return root; + } + + /** + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * + * @param resultSet + * @param allocator Memory allocator to use. + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets. + * @return Arrow Data Objects {@link VectorSchemaRoot} + * @throws SQLException + */ + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BaseAllocator allocator, Calendar calendar) throws SQLException, IOException { + Preconditions.checkNotNull(resultSet, "JDBC ResultSet object can not be null"); + Preconditions.checkNotNull(allocator, "Memory Allocator object can not be null"); + Preconditions.checkNotNull(calendar, "Calendar object can not be null"); + + VectorSchemaRoot root = VectorSchemaRoot.create( + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), calendar), allocator); + JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, calendar); + return root; + } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index e48255e313a..8621c9f1cd6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -88,431 +88,431 @@ */ public class JdbcToArrowUtils { - private static final int DEFAULT_BUFFER_SIZE = 256; - private static final int DEFAULT_STREAM_BUFFER_SIZE = 1024; - private static final int DEFAULT_CLOB_SUBSTRING_READ_SIZE = 256; - - /** - * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. - * - * This method currently performs following type mapping for JDBC SQL data types to corresponding Arrow data types. - * - * CHAR --> ArrowType.Utf8 - * NCHAR --> ArrowType.Utf8 - * VARCHAR --> ArrowType.Utf8 - * NVARCHAR --> ArrowType.Utf8 - * LONGVARCHAR --> ArrowType.Utf8 - * LONGNVARCHAR --> ArrowType.Utf8 - * NUMERIC --> ArrowType.Decimal(precision, scale) - * DECIMAL --> ArrowType.Decimal(precision, scale) - * BIT --> ArrowType.Bool - * TINYINT --> ArrowType.Int(8, signed) - * SMALLINT --> ArrowType.Int(16, signed) - * INTEGER --> ArrowType.Int(32, signed) - * BIGINT --> ArrowType.Int(64, signed) - * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) - * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) - * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) - * BINARY --> ArrowType.Binary - * VARBINARY --> ArrowType.Binary - * LONGVARBINARY --> ArrowType.Binary - * DATE --> ArrowType.Date(DateUnit.MILLISECOND) - * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) - * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) - * CLOB --> ArrowType.Utf8 - * BLOB --> ArrowType.Binary - * - * @param rsmd ResultSetMetaData - * @return {@link Schema} - * @throws SQLException - */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { - - Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); - Preconditions.checkNotNull(calendar, "Calendar object can't be null"); - - List fields = new ArrayList<>(); - int columnCount = rsmd.getColumnCount(); - for (int i = 1; i <= columnCount; i++) { - String columnName = rsmd.getColumnName(i); - switch (rsmd.getColumnType(i)) { - case Types.BOOLEAN: - case Types.BIT: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Bool()), null)); - break; - case Types.TINYINT: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(8, true)), null)); - break; - case Types.SMALLINT: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(16, true)), null)); - break; - case Types.INTEGER: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(32, true)), null)); - break; - case Types.BIGINT: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(64, true)), null)); - break; - case Types.NUMERIC: - case Types.DECIMAL: - int precision = rsmd.getPrecision(i); - int scale = rsmd.getScale(i); - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Decimal(precision, scale)), null)); - break; - case Types.REAL: - case Types.FLOAT: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); - break; - case Types.DOUBLE: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(DOUBLE)), null)); - break; - case Types.CHAR: - case Types.NCHAR: - case Types.VARCHAR: - case Types.NVARCHAR: - case Types.LONGVARCHAR: - case Types.LONGNVARCHAR: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); - break; - case Types.DATE: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)); - break; - case Types.TIME: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); - break; - case Types.TIMESTAMP: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar.getTimeZone().getID())), null)); - break; - case Types.BINARY: - case Types.VARBINARY: - case Types.LONGVARBINARY: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); - break; - case Types.ARRAY: - // TODO Need to handle this type -// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); - break; - case Types.CLOB: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); - break; - case Types.BLOB: - fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); - break; - - default: - // no-op, shouldn't get here - break; - } - } - - return new Schema(fields, null); + private static final int DEFAULT_BUFFER_SIZE = 256; + private static final int DEFAULT_STREAM_BUFFER_SIZE = 1024; + private static final int DEFAULT_CLOB_SUBSTRING_READ_SIZE = 256; + + /** + * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. + *

+ * This method currently performs following type mapping for JDBC SQL data types to corresponding Arrow data types. + *

+ * CHAR --> ArrowType.Utf8 + * NCHAR --> ArrowType.Utf8 + * VARCHAR --> ArrowType.Utf8 + * NVARCHAR --> ArrowType.Utf8 + * LONGVARCHAR --> ArrowType.Utf8 + * LONGNVARCHAR --> ArrowType.Utf8 + * NUMERIC --> ArrowType.Decimal(precision, scale) + * DECIMAL --> ArrowType.Decimal(precision, scale) + * BIT --> ArrowType.Bool + * TINYINT --> ArrowType.Int(8, signed) + * SMALLINT --> ArrowType.Int(16, signed) + * INTEGER --> ArrowType.Int(32, signed) + * BIGINT --> ArrowType.Int(64, signed) + * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) + * BINARY --> ArrowType.Binary + * VARBINARY --> ArrowType.Binary + * LONGVARBINARY --> ArrowType.Binary + * DATE --> ArrowType.Date(DateUnit.MILLISECOND) + * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) + * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) + * CLOB --> ArrowType.Utf8 + * BLOB --> ArrowType.Binary + * + * @param rsmd ResultSetMetaData + * @return {@link Schema} + * @throws SQLException + */ + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { + + Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); + + List fields = new ArrayList<>(); + int columnCount = rsmd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + String columnName = rsmd.getColumnName(i); + switch (rsmd.getColumnType(i)) { + case Types.BOOLEAN: + case Types.BIT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Bool()), null)); + break; + case Types.TINYINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(8, true)), null)); + break; + case Types.SMALLINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(16, true)), null)); + break; + case Types.INTEGER: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(32, true)), null)); + break; + case Types.BIGINT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Int(64, true)), null)); + break; + case Types.NUMERIC: + case Types.DECIMAL: + int precision = rsmd.getPrecision(i); + int scale = rsmd.getScale(i); + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Decimal(precision, scale)), null)); + break; + case Types.REAL: + case Types.FLOAT: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(SINGLE)), null)); + break; + case Types.DOUBLE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.FloatingPoint(DOUBLE)), null)); + break; + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); + break; + case Types.DATE: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Date(DateUnit.MILLISECOND)), null)); + break; + case Types.TIME: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Time(TimeUnit.MILLISECOND, 32)), null)); + break; + case Types.TIMESTAMP: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar.getTimeZone().getID())), null)); + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + case Types.ARRAY: +// TODO Need to handle this type +// fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Utf8()), null)); + break; + case Types.BLOB: + fields.add(new Field(columnName, FieldType.nullable(new ArrowType.Binary()), null)); + break; + + default: + // no-op, shouldn't get here + break; + } } - private static void allocateVectors(VectorSchemaRoot root, int size) { - List vectors = root.getFieldVectors(); - for (FieldVector fieldVector: vectors) { - if (fieldVector instanceof BaseFixedWidthVector) { - ((BaseFixedWidthVector) fieldVector).allocateNew(size); - } else { - fieldVector.allocateNew(); - } - fieldVector.setInitialCapacity(size); - } + return new Schema(fields, null); + } + + private static void allocateVectors(VectorSchemaRoot root, int size) { + List vectors = root.getFieldVectors(); + for (FieldVector fieldVector : vectors) { + if (fieldVector instanceof BaseFixedWidthVector) { + ((BaseFixedWidthVector) fieldVector).allocateNew(size); + } else { + fieldVector.allocateNew(); + } + fieldVector.setInitialCapacity(size); } - - /** - * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate - * the given Arrow Vector objects. - * - * @param rs ResultSet to use to fetch the data from underlying database - * @param root Arrow {@link VectorSchemaRoot} object to populate - * @throws SQLException - */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException, IOException { - - Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); - Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); - Preconditions.checkNotNull(calendar, "Calendar object can't be null"); - - ResultSetMetaData rsmd = rs.getMetaData(); - int columnCount = rsmd.getColumnCount(); - - allocateVectors(root, DEFAULT_BUFFER_SIZE); - - int rowCount = 0; - while (rs.next()) { - for (int i = 1; i <= columnCount; i++) { - String columnName = rsmd.getColumnName(i); - switch (rsmd.getColumnType(i)) { - case Types.BOOLEAN: - case Types.BIT: - updateVector((BitVector)root.getVector(columnName), - rs.getBoolean(i), !rs.wasNull(), rowCount); - break; - case Types.TINYINT: - updateVector((TinyIntVector)root.getVector(columnName), - rs.getInt(i), !rs.wasNull(), rowCount); - break; - case Types.SMALLINT: - updateVector((SmallIntVector)root.getVector(columnName), - rs.getInt(i), !rs.wasNull(), rowCount); - break; - case Types.INTEGER: - updateVector((IntVector)root.getVector(columnName), - rs.getInt(i), !rs.wasNull(), rowCount); - break; - case Types.BIGINT: - updateVector((BigIntVector)root.getVector(columnName), - rs.getLong(i), !rs.wasNull(), rowCount); - break; - case Types.NUMERIC: - case Types.DECIMAL: - updateVector((DecimalVector)root.getVector(columnName), - rs.getBigDecimal(i), !rs.wasNull(), rowCount); - break; - case Types.REAL: - case Types.FLOAT: - updateVector((Float4Vector)root.getVector(columnName), - rs.getFloat(i), !rs.wasNull(), rowCount); - break; - case Types.DOUBLE: - updateVector((Float8Vector)root.getVector(columnName), - rs.getDouble(i), !rs.wasNull(), rowCount); - break; - case Types.CHAR: - case Types.NCHAR: - case Types.VARCHAR: - case Types.NVARCHAR: - case Types.LONGVARCHAR: - case Types.LONGNVARCHAR: - updateVector((VarCharVector)root.getVector(columnName), - rs.getString(i), !rs.wasNull(), rowCount); - break; - case Types.DATE: - updateVector((DateMilliVector) root.getVector(columnName), - rs.getDate(i, calendar), !rs.wasNull(), rowCount); - break; - case Types.TIME: - updateVector((TimeMilliVector) root.getVector(columnName), - rs.getTime(i, calendar), !rs.wasNull(), rowCount); - break; - case Types.TIMESTAMP: - // TODO: Need to handle precision such as milli, micro, nano - updateVector((TimeStampVector)root.getVector(columnName), - rs.getTimestamp(i, calendar), !rs.wasNull(), rowCount); - break; - case Types.BINARY: - case Types.VARBINARY: - case Types.LONGVARBINARY: - updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBinaryStream(i), !rs.wasNull(), rowCount); - break; - case Types.ARRAY: - // TODO Need to handle this type - // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); - break; - case Types.CLOB: - updateVector((VarCharVector)root.getVector(columnName), - rs.getClob(i), !rs.wasNull(), rowCount); - break; - case Types.BLOB: - updateVector((VarBinaryVector)root.getVector(columnName), - rs.getBlob(i), !rs.wasNull(), rowCount); - break; - - default: - // no-op, shouldn't get here - break; - } - } - rowCount++; + } + + /** + * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate + * the given Arrow Vector objects. + * + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate + * @throws SQLException + */ + public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) throws SQLException, IOException { + + Preconditions.checkNotNull(rs, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(root, "JDBC ResultSet object can't be null"); + Preconditions.checkNotNull(calendar, "Calendar object can't be null"); + + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + + allocateVectors(root, DEFAULT_BUFFER_SIZE); + + int rowCount = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + String columnName = rsmd.getColumnName(i); + switch (rsmd.getColumnType(i)) { + case Types.BOOLEAN: + case Types.BIT: + updateVector((BitVector) root.getVector(columnName), + rs.getBoolean(i), !rs.wasNull(), rowCount); + break; + case Types.TINYINT: + updateVector((TinyIntVector) root.getVector(columnName), + rs.getInt(i), !rs.wasNull(), rowCount); + break; + case Types.SMALLINT: + updateVector((SmallIntVector) root.getVector(columnName), + rs.getInt(i), !rs.wasNull(), rowCount); + break; + case Types.INTEGER: + updateVector((IntVector) root.getVector(columnName), + rs.getInt(i), !rs.wasNull(), rowCount); + break; + case Types.BIGINT: + updateVector((BigIntVector) root.getVector(columnName), + rs.getLong(i), !rs.wasNull(), rowCount); + break; + case Types.NUMERIC: + case Types.DECIMAL: + updateVector((DecimalVector) root.getVector(columnName), + rs.getBigDecimal(i), !rs.wasNull(), rowCount); + break; + case Types.REAL: + case Types.FLOAT: + updateVector((Float4Vector) root.getVector(columnName), + rs.getFloat(i), !rs.wasNull(), rowCount); + break; + case Types.DOUBLE: + updateVector((Float8Vector) root.getVector(columnName), + rs.getDouble(i), !rs.wasNull(), rowCount); + break; + case Types.CHAR: + case Types.NCHAR: + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + updateVector((VarCharVector) root.getVector(columnName), + rs.getString(i), !rs.wasNull(), rowCount); + break; + case Types.DATE: + updateVector((DateMilliVector) root.getVector(columnName), + rs.getDate(i, calendar), !rs.wasNull(), rowCount); + break; + case Types.TIME: + updateVector((TimeMilliVector) root.getVector(columnName), + rs.getTime(i, calendar), !rs.wasNull(), rowCount); + break; + case Types.TIMESTAMP: + // TODO: Need to handle precision such as milli, micro, nano + updateVector((TimeStampVector) root.getVector(columnName), + rs.getTimestamp(i, calendar), !rs.wasNull(), rowCount); + break; + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + updateVector((VarBinaryVector) root.getVector(columnName), + rs.getBinaryStream(i), !rs.wasNull(), rowCount); + break; + case Types.ARRAY: + // TODO Need to handle this type + // fields.add(new Field("list", FieldType.nullable(new ArrowType.List()), null)); + break; + case Types.CLOB: + updateVector((VarCharVector) root.getVector(columnName), + rs.getClob(i), !rs.wasNull(), rowCount); + break; + case Types.BLOB: + updateVector((VarBinaryVector) root.getVector(columnName), + rs.getBlob(i), !rs.wasNull(), rowCount); + break; + + default: + // no-op, shouldn't get here + break; } - root.setRowCount(rowCount); + } + rowCount++; } - - private static void updateVector(BitVector bitVector, boolean value, boolean isNonNull, int rowCount) { - NullableBitHolder holder = new NullableBitHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = value ? 1 : 0; - } - bitVector.setSafe(rowCount, holder); - bitVector.setValueCount(rowCount + 1); + root.setRowCount(rowCount); + } + + private static void updateVector(BitVector bitVector, boolean value, boolean isNonNull, int rowCount) { + NullableBitHolder holder = new NullableBitHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = value ? 1 : 0; } - - private static void updateVector(TinyIntVector tinyIntVector, int value, boolean isNonNull, int rowCount) { - NullableTinyIntHolder holder = new NullableTinyIntHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = (byte) value; - } - tinyIntVector.setSafe(rowCount, holder); - tinyIntVector.setValueCount(rowCount + 1); + bitVector.setSafe(rowCount, holder); + bitVector.setValueCount(rowCount + 1); + } + + private static void updateVector(TinyIntVector tinyIntVector, int value, boolean isNonNull, int rowCount) { + NullableTinyIntHolder holder = new NullableTinyIntHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = (byte) value; } - - private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { - NullableSmallIntHolder holder = new NullableSmallIntHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = (short) value; - } - smallIntVector.setSafe(rowCount, holder); - smallIntVector.setValueCount(rowCount + 1); + tinyIntVector.setSafe(rowCount, holder); + tinyIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(SmallIntVector smallIntVector, int value, boolean isNonNull, int rowCount) { + NullableSmallIntHolder holder = new NullableSmallIntHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = (short) value; } - - private static void updateVector(IntVector intVector, int value, boolean isNonNull, int rowCount) { - NullableIntHolder holder = new NullableIntHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = value; - } - intVector.setSafe(rowCount, holder); - intVector.setValueCount(rowCount + 1); + smallIntVector.setSafe(rowCount, holder); + smallIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(IntVector intVector, int value, boolean isNonNull, int rowCount) { + NullableIntHolder holder = new NullableIntHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = value; } - - private static void updateVector(BigIntVector bigIntVector, long value, boolean isNonNull, int rowCount) { - NullableBigIntHolder holder = new NullableBigIntHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = value; - } - bigIntVector.setSafe(rowCount, holder); - bigIntVector.setValueCount(rowCount + 1); + intVector.setSafe(rowCount, holder); + intVector.setValueCount(rowCount + 1); + } + + private static void updateVector(BigIntVector bigIntVector, long value, boolean isNonNull, int rowCount) { + NullableBigIntHolder holder = new NullableBigIntHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = value; } - - private static void updateVector(DecimalVector decimalVector, BigDecimal value, boolean isNonNull, int rowCount) { - NullableDecimalHolder holder = new NullableDecimalHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.precision = value.precision(); - holder.scale = value.scale(); - holder.buffer = decimalVector.getAllocator().buffer(DEFAULT_BUFFER_SIZE); - holder.start = 0; - DecimalUtility.writeBigDecimalToArrowBuf(value, holder.buffer, holder.start); - } - decimalVector.setSafe(rowCount, holder); - decimalVector.setValueCount(rowCount + 1); + bigIntVector.setSafe(rowCount, holder); + bigIntVector.setValueCount(rowCount + 1); + } + + private static void updateVector(DecimalVector decimalVector, BigDecimal value, boolean isNonNull, int rowCount) { + NullableDecimalHolder holder = new NullableDecimalHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.precision = value.precision(); + holder.scale = value.scale(); + holder.buffer = decimalVector.getAllocator().buffer(DEFAULT_BUFFER_SIZE); + holder.start = 0; + DecimalUtility.writeBigDecimalToArrowBuf(value, holder.buffer, holder.start); } - - private static void updateVector(Float4Vector float4Vector, float value, boolean isNonNull, int rowCount) { - NullableFloat4Holder holder = new NullableFloat4Holder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = value; - } - float4Vector.setSafe(rowCount, holder); - float4Vector.setValueCount(rowCount + 1); + decimalVector.setSafe(rowCount, holder); + decimalVector.setValueCount(rowCount + 1); + } + + private static void updateVector(Float4Vector float4Vector, float value, boolean isNonNull, int rowCount) { + NullableFloat4Holder holder = new NullableFloat4Holder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = value; } - - private static void updateVector(Float8Vector float8Vector, double value, boolean isNonNull, int rowCount) { - NullableFloat8Holder holder = new NullableFloat8Holder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = value; - } - float8Vector.setSafe(rowCount, holder); - float8Vector.setValueCount(rowCount + 1); + float4Vector.setSafe(rowCount, holder); + float4Vector.setValueCount(rowCount + 1); + } + + private static void updateVector(Float8Vector float8Vector, double value, boolean isNonNull, int rowCount) { + NullableFloat8Holder holder = new NullableFloat8Holder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = value; } - - private static void updateVector(VarCharVector varcharVector, String value, boolean isNonNull, int rowCount) { - NullableVarCharHolder holder = new NullableVarCharHolder(); - holder.isSet = isNonNull? 1: 0; - varcharVector.setIndexDefined(rowCount); - if (isNonNull) { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - holder.buffer = varcharVector.getAllocator().buffer(bytes.length); - holder.buffer.setBytes(0, bytes, 0, bytes.length); - holder.start = 0; - holder.end = bytes.length; - } else { - holder.buffer = varcharVector.getAllocator().buffer(0); - } - varcharVector.setSafe(rowCount, holder); - varcharVector.setValueCount(rowCount + 1); + float8Vector.setSafe(rowCount, holder); + float8Vector.setValueCount(rowCount + 1); + } + + private static void updateVector(VarCharVector varcharVector, String value, boolean isNonNull, int rowCount) { + NullableVarCharHolder holder = new NullableVarCharHolder(); + holder.isSet = isNonNull ? 1 : 0; + varcharVector.setIndexDefined(rowCount); + if (isNonNull) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + holder.buffer = varcharVector.getAllocator().buffer(bytes.length); + holder.buffer.setBytes(0, bytes, 0, bytes.length); + holder.start = 0; + holder.end = bytes.length; + } else { + holder.buffer = varcharVector.getAllocator().buffer(0); } - - private static void updateVector(DateMilliVector dateMilliVector, Date date, boolean isNonNull, int rowCount) { - NullableDateMilliHolder holder = new NullableDateMilliHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull) { - holder.value = date.getTime(); - } - dateMilliVector.setSafe(rowCount, holder); - dateMilliVector.setValueCount(rowCount + 1); + varcharVector.setSafe(rowCount, holder); + varcharVector.setValueCount(rowCount + 1); + } + + private static void updateVector(DateMilliVector dateMilliVector, Date date, boolean isNonNull, int rowCount) { + NullableDateMilliHolder holder = new NullableDateMilliHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull) { + holder.value = date.getTime(); } - - private static void updateVector(TimeMilliVector timeMilliVector, Time time, boolean isNonNull, int rowCount) { - NullableTimeMilliHolder holder = new NullableTimeMilliHolder(); - holder.isSet = isNonNull? 1: 0; - if (isNonNull && time != null) { - holder.value = (int)time.getTime(); - } - timeMilliVector.setSafe(rowCount, holder); - timeMilliVector.setValueCount(rowCount + 1); + dateMilliVector.setSafe(rowCount, holder); + dateMilliVector.setValueCount(rowCount + 1); + } + + private static void updateVector(TimeMilliVector timeMilliVector, Time time, boolean isNonNull, int rowCount) { + NullableTimeMilliHolder holder = new NullableTimeMilliHolder(); + holder.isSet = isNonNull ? 1 : 0; + if (isNonNull && time != null) { + holder.value = (int) time.getTime(); } - - private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, boolean isNonNull, int rowCount) { - //TODO: Need to handle precision such as milli, micro, nano - timeStampVector.setValueCount(rowCount + 1); - if (timestamp != null) { - timeStampVector.setSafe(rowCount, timestamp.getTime()); - } else { - timeStampVector.setNull(rowCount); - } + timeMilliVector.setSafe(rowCount, holder); + timeMilliVector.setValueCount(rowCount + 1); + } + + private static void updateVector(TimeStampVector timeStampVector, Timestamp timestamp, boolean isNonNull, int rowCount) { + //TODO: Need to handle precision such as milli, micro, nano + timeStampVector.setValueCount(rowCount + 1); + if (timestamp != null) { + timeStampVector.setSafe(rowCount, timestamp.getTime()); + } else { + timeStampVector.setNull(rowCount); } - - private static void updateVector(VarBinaryVector varBinaryVector, InputStream is, boolean isNonNull, int rowCount) throws IOException { - varBinaryVector.setValueCount(rowCount + 1); - if (isNonNull && is != null) { - VarBinaryHolder holder = new VarBinaryHolder(); - ArrowBuf arrowBuf = varBinaryVector.getDataBuffer(); - holder.start = 0; - byte[] bytes = new byte[DEFAULT_STREAM_BUFFER_SIZE]; - int total = 0; - while (true) { - int read = is.read(bytes, 0, DEFAULT_STREAM_BUFFER_SIZE); - if (read == -1) { - break; - } - arrowBuf.setBytes(total, bytes, total, read); - total += read; - } - holder.end = total; - holder.buffer = arrowBuf; - varBinaryVector.set(rowCount, holder); - varBinaryVector.setIndexDefined(rowCount); - } else { - varBinaryVector.setNull(rowCount); + } + + private static void updateVector(VarBinaryVector varBinaryVector, InputStream is, boolean isNonNull, int rowCount) throws IOException { + varBinaryVector.setValueCount(rowCount + 1); + if (isNonNull && is != null) { + VarBinaryHolder holder = new VarBinaryHolder(); + ArrowBuf arrowBuf = varBinaryVector.getDataBuffer(); + holder.start = 0; + byte[] bytes = new byte[DEFAULT_STREAM_BUFFER_SIZE]; + int total = 0; + while (true) { + int read = is.read(bytes, 0, DEFAULT_STREAM_BUFFER_SIZE); + if (read == -1) { + break; } + arrowBuf.setBytes(total, bytes, total, read); + total += read; + } + holder.end = total; + holder.buffer = arrowBuf; + varBinaryVector.set(rowCount, holder); + varBinaryVector.setIndexDefined(rowCount); + } else { + varBinaryVector.setNull(rowCount); } - - private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException, IOException { - varcharVector.setValueCount(rowCount + 1); - if (isNonNull && clob != null) { - VarCharHolder holder = new VarCharHolder(); - ArrowBuf arrowBuf = varcharVector.getDataBuffer(); - holder.start = 0; - long length = clob.length(); - int read = 1; - int readSize = length < DEFAULT_CLOB_SUBSTRING_READ_SIZE? (int)length: DEFAULT_CLOB_SUBSTRING_READ_SIZE; - int totalBytes = 0; - while (read <= length) { - String str = clob.getSubString(read, readSize); - byte[] bytes = str.getBytes(StandardCharsets.UTF_8); - arrowBuf.setBytes(totalBytes, new ByteArrayInputStream(bytes, 0, bytes.length), bytes.length); - totalBytes += bytes.length; - read += readSize; - } - holder.end = totalBytes; - holder.buffer = arrowBuf; - varcharVector.set(rowCount, holder); - varcharVector.setIndexDefined(rowCount); - } else { - varcharVector.setNull(rowCount); - } + } + + private static void updateVector(VarCharVector varcharVector, Clob clob, boolean isNonNull, int rowCount) throws SQLException, IOException { + varcharVector.setValueCount(rowCount + 1); + if (isNonNull && clob != null) { + VarCharHolder holder = new VarCharHolder(); + ArrowBuf arrowBuf = varcharVector.getDataBuffer(); + holder.start = 0; + long length = clob.length(); + int read = 1; + int readSize = length < DEFAULT_CLOB_SUBSTRING_READ_SIZE ? (int) length : DEFAULT_CLOB_SUBSTRING_READ_SIZE; + int totalBytes = 0; + while (read <= length) { + String str = clob.getSubString(read, readSize); + byte[] bytes = str.getBytes(StandardCharsets.UTF_8); + arrowBuf.setBytes(totalBytes, new ByteArrayInputStream(bytes, 0, bytes.length), bytes.length); + totalBytes += bytes.length; + read += readSize; + } + holder.end = totalBytes; + holder.buffer = arrowBuf; + varcharVector.set(rowCount, holder); + varcharVector.setIndexDefined(rowCount); + } else { + varcharVector.setNull(rowCount); } + } - private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException, IOException { - updateVector(varBinaryVector, blob != null ? blob.getBinaryStream() : null, isNonNull, rowCount); - } + private static void updateVector(VarBinaryVector varBinaryVector, Blob blob, boolean isNonNull, int rowCount) throws SQLException, IOException { + updateVector(varBinaryVector, blob != null ? blob.getBinaryStream() : null, isNonNull, rowCount); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 19438c270dd..102824777ad 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,6 +23,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; + import org.apache.arrow.vector.VectorSchemaRoot; import org.junit.After; import org.junit.Before; @@ -34,80 +35,86 @@ * Class to abstract out some common test functionality for testing JDBC to Arrow. */ public abstract class AbstractJdbcToArrowTest { - protected Connection conn = null; - protected Table table; + protected Connection conn = null; + protected Table table; - /** - * This method creates Table object after reading YAML file - * @param ymlFilePath - * @return - * @throws IOException - */ - protected static Table getTable(String ymlFilePath, Class clss) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( + /** + * This method creates Table object after reading YAML file + * + * @param ymlFilePath + * @return + * @throws IOException + */ + protected static Table getTable(String ymlFilePath, Class clss) throws IOException { + return new ObjectMapper(new YAMLFactory()).readValue( clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + } + + + /** + * This method creates Connection object and DB table and also populate data into table for test + * + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } } + } - - /** - * This method creates Connection object and DB table and also populate data into table for test - * @throws SQLException - * @throws ClassNotFoundException - */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } + /** + * Clean up method to close connection after test completes + * + * @throws SQLException + */ + @After + public void destroy() throws SQLException { + if (conn != null) { + conn.close(); + conn = null; } + } - /** - * Clean up method to close connection after test completes - * @throws SQLException - */ - @After - public void destroy() throws SQLException { - if (conn != null) { - conn.close(); - conn = null; - } + /** + * This method returns collection of Table object for each test iteration + * + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + public static Object[][] prepareTestData(String[] testFiles, Class clss) throws SQLException, ClassNotFoundException, IOException { + Object[][] tableArr = new Object[testFiles.length][]; + int i = 0; + for (String testFile : testFiles) { + tableArr[i++] = new Object[]{getTable(testFile, clss)}; } + return tableArr; + } - /** - * This method returns collection of Table object for each test iteration - * @return - * @throws SQLException - * @throws ClassNotFoundException - * @throws IOException - */ - public static Object[][] prepareTestData(String[] testFiles, Class clss) throws SQLException, ClassNotFoundException, IOException { - Object[][] tableArr = new Object[testFiles.length][]; - int i = 0; - for (String testFile: testFiles) { - tableArr[i++] = new Object[]{getTable(testFile, clss)}; - } - return tableArr; - } + /** + * Abstract method to implement test Functionality to test JdbcToArrow methods + * + * @throws SQLException + * @throws IOException + */ + @Test + public abstract void testJdbcToArroValues() throws SQLException, IOException; - /** - * Abstract method to implement test Functionality to test JdbcToArrow methods - * @throws SQLException - * @throws IOException - */ - @Test - public abstract void testJdbcToArroValues() throws SQLException, IOException; - - /** - * Abstract method to implement logic to assert test various datatype values - * @param root - */ - public abstract void testDataSets(VectorSchemaRoot root); + /** + * Abstract method to implement logic to assert test various datatype values + * + * @param root + */ + public abstract void testDataSets(VectorSchemaRoot root); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 8fabc4dcdae..71a829d71f7 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import java.math.BigDecimal; import java.nio.charset.Charset; + import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -34,6 +35,7 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; @@ -41,239 +43,238 @@ /** * This is a Helper class which has functionalities to read and assert the values from the given FieldVector object - * */ public class JdbcToArrowTestHelper { - public static void assertIntVectorValues(IntVector intVector, int rowCount, Integer[] values) { - assertEquals(rowCount, intVector.getValueCount()); + public static void assertIntVectorValues(IntVector intVector, int rowCount, Integer[] values) { + assertEquals(rowCount, intVector.getValueCount()); - for(int j = 0; j < intVector.getValueCount(); j++) { - assertEquals(values[j].intValue(), intVector.get(j)); - } + for (int j = 0; j < intVector.getValueCount(); j++) { + assertEquals(values[j].intValue(), intVector.get(j)); } + } - public static void assertBooleanVectorValues(BitVector bitVector, int rowCount, Boolean[] values){ - assertEquals(rowCount, bitVector.getValueCount()); + public static void assertBooleanVectorValues(BitVector bitVector, int rowCount, Boolean[] values) { + assertEquals(rowCount, bitVector.getValueCount()); - for(int j = 0; j < bitVector.getValueCount(); j++){ - assertEquals(values[j].booleanValue(), bitVector.get(j) == 1); - } + for (int j = 0; j < bitVector.getValueCount(); j++) { + assertEquals(values[j].booleanValue(), bitVector.get(j) == 1); } + } - public static void assertBitVectorValues(BitVector bitVector, int rowCount, Integer[] values){ - assertEquals(rowCount, bitVector.getValueCount()); + public static void assertBitVectorValues(BitVector bitVector, int rowCount, Integer[] values) { + assertEquals(rowCount, bitVector.getValueCount()); - for(int j = 0; j < bitVector.getValueCount(); j++){ - assertEquals(values[j].intValue(), bitVector.get(j)); - } + for (int j = 0; j < bitVector.getValueCount(); j++) { + assertEquals(values[j].intValue(), bitVector.get(j)); } + } - public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, Integer[] values){ - assertEquals(rowCount, tinyIntVector.getValueCount()); + public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, Integer[] values) { + assertEquals(rowCount, tinyIntVector.getValueCount()); - for(int j = 0; j < tinyIntVector.getValueCount(); j++){ - assertEquals(values[j].intValue(), tinyIntVector.get(j)); - } + for (int j = 0; j < tinyIntVector.getValueCount(); j++) { + assertEquals(values[j].intValue(), tinyIntVector.get(j)); } + } - public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, Integer[] values){ - assertEquals(rowCount, smallIntVector.getValueCount()); + public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, Integer[] values) { + assertEquals(rowCount, smallIntVector.getValueCount()); - for(int j = 0; j < smallIntVector.getValueCount(); j++){ - assertEquals(values[j].intValue(), smallIntVector.get(j)); - } + for (int j = 0; j < smallIntVector.getValueCount(); j++) { + assertEquals(values[j].intValue(), smallIntVector.get(j)); } + } - public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, Long[] values){ - assertEquals(rowCount, bigIntVector.getValueCount()); + public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, Long[] values) { + assertEquals(rowCount, bigIntVector.getValueCount()); - for(int j = 0; j < bigIntVector.getValueCount(); j++){ - assertEquals(values[j].longValue(), bigIntVector.get(j)); - } + for (int j = 0; j < bigIntVector.getValueCount(); j++) { + assertEquals(values[j].longValue(), bigIntVector.get(j)); } + } - public static void assertDecimalVectorValues(DecimalVector decimalVector, int rowCount, BigDecimal[] values){ - assertEquals(rowCount, decimalVector.getValueCount()); + public static void assertDecimalVectorValues(DecimalVector decimalVector, int rowCount, BigDecimal[] values) { + assertEquals(rowCount, decimalVector.getValueCount()); - for(int j = 0; j < decimalVector.getValueCount(); j++){ - assertNotNull(decimalVector.getObject(j)); - assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); - } + for (int j = 0; j < decimalVector.getValueCount(); j++) { + assertNotNull(decimalVector.getObject(j)); + assertEquals(values[j].doubleValue(), decimalVector.getObject(j).doubleValue(), 0); } + } - public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, Double[] values){ - assertEquals(rowCount, float8Vector.getValueCount()); + public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, Double[] values) { + assertEquals(rowCount, float8Vector.getValueCount()); - for(int j = 0; j < float8Vector.getValueCount(); j++){ - assertEquals(values[j], float8Vector.get(j), 0.01); - } + for (int j = 0; j < float8Vector.getValueCount(); j++) { + assertEquals(values[j], float8Vector.get(j), 0.01); } + } - public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, Float[] values){ - assertEquals(rowCount, float4Vector.getValueCount()); + public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, Float[] values) { + assertEquals(rowCount, float4Vector.getValueCount()); - for(int j = 0; j < float4Vector.getValueCount(); j++){ - assertEquals(values[j], float4Vector.get(j), 0.01); - } + for (int j = 0; j < float4Vector.getValueCount(); j++) { + assertEquals(values[j], float4Vector.get(j), 0.01); } + } - public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, Long[] values){ - assertEquals(rowCount, timeMilliVector.getValueCount()); + public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, Long[] values) { + assertEquals(rowCount, timeMilliVector.getValueCount()); - for(int j = 0; j < timeMilliVector.getValueCount(); j++){ - assertEquals(values[j].longValue(), timeMilliVector.get(j)); - } + for (int j = 0; j < timeMilliVector.getValueCount(); j++) { + assertEquals(values[j].longValue(), timeMilliVector.get(j)); } + } - public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, Long[] values){ - assertEquals(rowCount, dateMilliVector.getValueCount()); + public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, Long[] values) { + assertEquals(rowCount, dateMilliVector.getValueCount()); - for(int j = 0; j < dateMilliVector.getValueCount(); j++){ - assertEquals(values[j].longValue(), dateMilliVector.get(j)); - } + for (int j = 0; j < dateMilliVector.getValueCount(); j++) { + assertEquals(values[j].longValue(), dateMilliVector.get(j)); } + } - public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, Long[] values){ - assertEquals(rowCount, timeStampVector.getValueCount()); + public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, Long[] values) { + assertEquals(rowCount, timeStampVector.getValueCount()); - for(int j = 0; j < timeStampVector.getValueCount(); j++){ - assertEquals(values[j].longValue(), timeStampVector.get(j)); - } + for (int j = 0; j < timeStampVector.getValueCount(); j++) { + assertEquals(values[j].longValue(), timeStampVector.get(j)); } + } - public static void assertVarBinaryVectorValues (VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { - assertEquals(rowCount, varBinaryVector.getValueCount()); + public static void assertVarBinaryVectorValues(VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { + assertEquals(rowCount, varBinaryVector.getValueCount()); - for(int j = 0; j < varBinaryVector.getValueCount(); j++){ - assertArrayEquals(values[j], varBinaryVector.get(j)); - } + for (int j = 0; j < varBinaryVector.getValueCount(); j++) { + assertArrayEquals(values[j], varBinaryVector.get(j)); } + } - public static void assertVarcharVectorValues(VarCharVector varCharVector, int rowCount, byte[][] values) { - assertEquals(rowCount, varCharVector.getValueCount()); + public static void assertVarcharVectorValues(VarCharVector varCharVector, int rowCount, byte[][] values) { + assertEquals(rowCount, varCharVector.getValueCount()); - for(int j = 0; j < varCharVector.getValueCount(); j++){ - assertArrayEquals(values[j], varCharVector.get(j)); - } + for (int j = 0; j < varCharVector.getValueCount(); j++) { + assertArrayEquals(values[j], varCharVector.get(j)); } + } - public static void assertNullValues(BaseValueVector vector, int rowCount) { - assertEquals(rowCount, vector.getValueCount()); + public static void assertNullValues(BaseValueVector vector, int rowCount) { + assertEquals(rowCount, vector.getValueCount()); - for(int j = 0; j < vector.getValueCount(); j++) { - assertTrue(vector.isNull(j)); - } + for (int j = 0; j < vector.getValueCount(); j++) { + assertTrue(vector.isNull(j)); } - - public static byte[] hexStringToByteArray(String s) { - int len = s.length(); - byte[] data = new byte[len / 2]; - for (int i = 0; i < len; i += 2) { - data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) - + Character.digit(s.charAt(i+1), 16)); - } - return data; + } + + public static byte[] hexStringToByteArray(String s) { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + + Character.digit(s.charAt(i + 1), 16)); } - - public static Integer [] getIntValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Integer [] valueArr = new Integer [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Integer.parseInt(data); - } - return valueArr; + return data; + } + + public static Integer[] getIntValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + Integer[] valueArr = new Integer[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = Integer.parseInt(data); } - - public static Boolean [] getBooleanValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Boolean [] valueArr = new Boolean [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().equals("1"); - } - return valueArr; + return valueArr; + } + + public static Boolean[] getBooleanValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + Boolean[] valueArr = new Boolean[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = data.trim().equals("1"); } - - public static BigDecimal [] getDecimalValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - BigDecimal [] valueArr = new BigDecimal [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr[i++] = new BigDecimal(data); - } - return valueArr; + return valueArr; + } + + public static BigDecimal[] getDecimalValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + BigDecimal[] valueArr = new BigDecimal[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = new BigDecimal(data); } - - public static Double [] getDoubleValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Double [] valueArr = new Double [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Double.parseDouble(data); - } - return valueArr; + return valueArr; + } + + public static Double[] getDoubleValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + Double[] valueArr = new Double[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = Double.parseDouble(data); } - - public static Float [] getFloatValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Float [] valueArr = new Float [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Float.parseFloat(data); - } - return valueArr; + return valueArr; + } + + public static Float[] getFloatValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + Float[] valueArr = new Float[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = Float.parseFloat(data); } - - public static Long [] getLongValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - Long [] valueArr = new Long [dataArr.length]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = Long.parseLong(data); - } - return valueArr; + return valueArr; + } + + public static Long[] getLongValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + Long[] valueArr = new Long[dataArr.length]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = Long.parseLong(data); } - - public static byte [][] getCharArray(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().getBytes(); - } - return valueArr; + return valueArr; + } + + public static byte[][] getCharArray(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + byte[][] valueArr = new byte[dataArr.length][]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = data.trim().getBytes(); } - - public static byte [][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = data.trim().getBytes(charSet); - } - return valueArr; + return valueArr; + } + + public static byte[][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) { + String[] dataArr = getValues(values, dataType); + byte[][] valueArr = new byte[dataArr.length][]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = data.trim().getBytes(charSet); } - - public static byte [][] getBinaryValues(String[] values, String dataType) { - String[] dataArr= getValues(values, dataType); - byte [][] valueArr = new byte [dataArr.length][]; - int i =0; - for(String data : dataArr) { - valueArr [i++] = hexStringToByteArray(data.trim()); - } - return valueArr; - } - - public static String [] getValues(String[] values, String dataType) { - String value = ""; - for(String val : values) { - if(val.startsWith(dataType)) { - value = val.split("=")[1]; - break; - } - } - return value.split(","); + return valueArr; + } + + public static byte[][] getBinaryValues(String[] values, String dataType) { + String[] dataArr = getValues(values, dataType); + byte[][] valueArr = new byte[dataArr.length][]; + int i = 0; + for (String data : dataArr) { + valueArr[i++] = hexStringToByteArray(data.trim()); + } + return valueArr; + } + + public static String[] getValues(String[] values, String dataType) { + String value = ""; + for (String val : values) { + if (val.startsWith(dataType)) { + value = val.split("=")[1]; + break; + } } + return value.split(","); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index b4305eb38b9..5bfdf756403 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -28,208 +28,208 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public class Table { - private String name; - private String type; - private String vector; - private String timezone; - private String create; - private String[] data; - private String query; - private String drop; - private String[] values; - private String[] vectors; - private int rowCount; - - public Table() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getVector() { - return vector; - } + private String name; + private String type; + private String vector; + private String timezone; + private String create; + private String[] data; + private String query; + private String drop; + private String[] values; + private String[] vectors; + private int rowCount; + + public Table() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getVector() { + return vector; + } + + public void setVector(String vector) { + this.vector = vector; + } + + public String[] getValues() { + return values; + } + + public void setValues(String[] values) { + this.values = values; + } + + public Long[] getLongValues() { + Long[] arr = new Long[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = Long.parseLong(str); + } + return arr; + } + + public Integer[] getIntValues() { + Integer[] arr = new Integer[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = Integer.parseInt(str); + } + return arr; + } + + public Boolean[] getBoolValues() { + Boolean[] arr = new Boolean[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = Boolean.parseBoolean(str); + } + return arr; + } + + public BigDecimal[] getBigDecimalValues() { + BigDecimal[] arr = new BigDecimal[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = new BigDecimal(str); + } + return arr; + } + + public Double[] getDoubleValues() { + Double[] arr = new Double[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = Double.parseDouble(str); + } + return arr; + } + + public Float[] getFloatValues() { + Float[] arr = new Float[values.length]; + int i = 0; + for (String str : values) { + arr[i++] = Float.parseFloat(str); + } + return arr; + } + + public byte[][] getBinaryValues() { + return getHexToByteArray(values); + } + + public byte[][] getVarCharValues() { + return getByteArray(values); + } + + public byte[][] getBlobValues() { + return getBinaryValues(); + } + + public byte[][] getClobValues() { + return getByteArray(values); + } + + public byte[][] getCharValues() { + return getByteArray(values); + } + + public String getCreate() { + return create; + } + + public void setCreate(String create) { + this.create = create; + } + + public String[] getData() { + return data; + } + + public void setData(String[] data) { + this.data = data; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public String getDrop() { + return drop; + } + + public void setDrop(String drop) { + this.drop = drop; + } - public void setVector(String vector) { - this.vector = vector; - } + public String getTimezone() { + return timezone; + } + + public void setTimezone(String timezone) { + this.timezone = timezone; + } - public String[] getValues() { - return values; - } + public String[] getVectors() { + return vectors; + } - public void setValues(String[] values) { - this.values = values; - } + public void setVectors(String[] vectors) { + this.vectors = vectors; + } - public Long[] getLongValues() { - Long[] arr = new Long[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = Long.parseLong(str); - } - return arr; - } + public int getRowCount() { + return rowCount; + } - public Integer[] getIntValues() { - Integer[] arr = new Integer[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = Integer.parseInt(str); - } - return arr; - } - - public Boolean[] getBoolValues() { - Boolean[] arr = new Boolean[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = Boolean.parseBoolean(str); - } - return arr; - } - - public BigDecimal[] getBigDecimalValues() { - BigDecimal[] arr = new BigDecimal[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = new BigDecimal(str); - } - return arr; - } - - public Double[] getDoubleValues() { - Double[] arr = new Double[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = Double.parseDouble(str); - } - return arr; - } - - public Float[] getFloatValues() { - Float[] arr = new Float[values.length]; - int i = 0; - for (String str: values) { - arr[i++] = Float.parseFloat(str); - } - return arr; - } - - public byte[][] getBinaryValues() { - return getHexToByteArray(values); - } - - public byte[][] getVarCharValues() { - return getByteArray(values); - } - - public byte[][] getBlobValues() { - return getBinaryValues(); - } - - public byte[][] getClobValues() { - return getByteArray(values); - } - - public byte[][] getCharValues() { - return getByteArray(values); - } - - public String getCreate() { - return create; - } - - public void setCreate(String create) { - this.create = create; - } - - public String[] getData() { - return data; - } - - public void setData(String[] data) { - this.data = data; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public String getDrop() { - return drop; - } - - public void setDrop(String drop) { - this.drop = drop; - } - - public String getTimezone() { - return timezone; - } - - public void setTimezone(String timezone) { - this.timezone = timezone; - } - - public String[] getVectors() { - return vectors; - } - - public void setVectors(String[] vectors) { - this.vectors = vectors; - } - - public int getRowCount() { - return rowCount; - } - - public void setRowCount(int rowCount) { - this.rowCount = rowCount; - } + public void setRowCount(int rowCount) { + this.rowCount = rowCount; + } - private byte[][] getByteArray(String[] data) { - byte[][] byteArr = new byte[data.length][]; + private byte[][] getByteArray(String[] data) { + byte[][] byteArr = new byte[data.length][]; - for (int i = 0; i < data.length; i++) { - byteArr[i] = data[i].getBytes(StandardCharsets.UTF_8); - } - return byteArr; + for (int i = 0; i < data.length; i++) { + byteArr[i] = data[i].getBytes(StandardCharsets.UTF_8); } + return byteArr; + } - private byte[][] getHexToByteArray(String[] data) { - byte[][] byteArr = new byte[data.length][]; + private byte[][] getHexToByteArray(String[] data) { + byte[][] byteArr = new byte[data.length][]; - for (int i = 0; i < data.length; i++) { - byteArr[i] = hexStringToByteArray(data[i]); - } - return byteArr; + for (int i = 0; i < data.length; i++) { + byteArr[i] = hexStringToByteArray(data[i]); } + return byteArr; + } - private static byte[] hexStringToByteArray(String s) { - int len = s.length(); - byte[] data = new byte[len / 2]; - for (int i = 0; i < len; i += 2) { - data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) - + Character.digit(s.charAt(i + 1), 16)); - } - return data; + private static byte[] hexStringToByteArray(String s) { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + + Character.digit(s.charAt(i + 1), 16)); } + return data; + } } \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java index db3620f38c7..473b8f70263 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -43,90 +43,92 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getCharArrayWithCharSet; /** - * - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with UTF-8 Charset, including + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with UTF-8 Charset, including * the multi-byte CJK characters for H2 database - * */ @RunWith(Parameterized.class) public class JdbcToArrowCharSetTest extends AbstractJdbcToArrowTest { - private static final String VARCHAR = "VARCHAR_FIELD13"; - private static final String CHAR = "CHAR_FIELD16"; - private static final String CLOB = "CLOB_FIELD15"; - - private static final String[] testFiles = { - "h2/test1_charset_h2.yml", - "h2/test1_charset_ch_h2.yml", - "h2/test1_charset_jp_h2.yml", - "h2/test1_charset_kr_h2.yml" - }; - - /** - * Constructor which populate table object for each test iteration - * @param table - */ - public JdbcToArrowCharSetTest (Table table) { - this.table = table; - } - - /** - * This method creates Connection object and DB table and also populate data into table for test - * @throws SQLException - * @throws ClassNotFoundException - */ - @Before - public void setUp() throws SQLException, ClassNotFoundException { - String url = "jdbc:h2:mem:JdbcToArrowTest?characterEncoding=UTF-8"; - String driver = "org.h2.Driver"; - Class.forName(driver); - conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { - stmt.executeUpdate(table.getCreate()); - for (String insert : table.getData()) { - stmt.executeUpdate(insert); - } - } - } + private static final String VARCHAR = "VARCHAR_FIELD13"; + private static final String CHAR = "CHAR_FIELD16"; + private static final String CLOB = "CLOB_FIELD15"; - /** - * This method returns collection of Table object for each test iteration - * @return - * @throws SQLException - * @throws ClassNotFoundException - * @throws IOException - */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.asList(prepareTestData(testFiles, JdbcToArrowCharSetTest.class)); - } - - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with UTF-8 Charset, including - * the multi-byte CJK characters - */ - @Test - public void testJdbcToArroValues() throws SQLException, IOException { - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); - } - - /** - * This method calls the assert methods for various DataSets - * @param root - */ - public void testDataSets(VectorSchemaRoot root) { - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), - getCharArrayWithCharSet(table.getValues(), CLOB, StandardCharsets.UTF_8)); + private static final String[] testFiles = { + "h2/test1_charset_h2.yml", + "h2/test1_charset_ch_h2.yml", + "h2/test1_charset_jp_h2.yml", + "h2/test1_charset_kr_h2.yml" + }; - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), - getCharArrayWithCharSet(table.getValues(), VARCHAR, StandardCharsets.UTF_8)); + /** + * Constructor which populate table object for each test iteration + * + * @param table + */ + public JdbcToArrowCharSetTest(Table table) { + this.table = table; + } - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), - getCharArrayWithCharSet(table.getValues(), CHAR, StandardCharsets.UTF_8)); + /** + * This method creates Connection object and DB table and also populate data into table for test + * + * @throws SQLException + * @throws ClassNotFoundException + */ + @Before + public void setUp() throws SQLException, ClassNotFoundException { + String url = "jdbc:h2:mem:JdbcToArrowTest?characterEncoding=UTF-8"; + String driver = "org.h2.Driver"; + Class.forName(driver); + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement();) { + stmt.executeUpdate(table.getCreate()); + for (String insert : table.getData()) { + stmt.executeUpdate(insert); + } } + } + + /** + * This method returns collection of Table object for each test iteration + * + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowCharSetTest.class)); + } + + /** + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with UTF-8 Charset, including + * the multi-byte CJK characters + */ + @Test + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + } + + /** + * This method calls the assert methods for various DataSets + * + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), CLOB, StandardCharsets.UTF_8)); + + assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), VARCHAR, StandardCharsets.UTF_8)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + getCharArrayWithCharSet(table.getValues(), CHAR, StandardCharsets.UTF_8)); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index c615d0b77d3..aa256643008 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -39,6 +39,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; + import java.io.IOException; import java.sql.SQLException; import java.util.Arrays; @@ -61,135 +62,139 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; /** - * - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database * using multiple test data files - * */ @RunWith(Parameterized.class) public class JdbcToArrowDataTypesTest extends AbstractJdbcToArrowTest { - private static final String BIGINT = "big_int"; - private static final String BINARY = "binary"; - private static final String BIT = "bit"; - private static final String BLOB = "blob"; - private static final String BOOL = "bool"; - private static final String CHAR = "char"; - private static final String CLOB = "clob"; - private static final String DATE = "date"; - private static final String DECIMAL = "decimal"; - private static final String DOUBLE = "double"; - private static final String INT = "int"; - private static final String REAL = "real"; - private static final String SMALLINT = "small_int"; - private static final String TIME = "time"; - private static final String TIMESTAMP = "timestamp"; - private static final String TINYINT = "tiny_int"; - private static final String VARCHAR = "varchar"; - - private static final String[] testFiles = { - "h2/test1_bigint_h2.yml", - "h2/test1_binary_h2.yml", - "h2/test1_bit_h2.yml", - "h2/test1_blob_h2.yml", - "h2/test1_bool_h2.yml", - "h2/test1_char_h2.yml", - "h2/test1_clob_h2.yml", - "h2/test1_date_h2.yml", - "h2/test1_decimal_h2.yml", - "h2/test1_double_h2.yml", - "h2/test1_int_h2.yml", - "h2/test1_real_h2.yml", - "h2/test1_smallint_h2.yml", - "h2/test1_time_h2.yml", - "h2/test1_timestamp_h2.yml", - "h2/test1_tinyint_h2.yml", - "h2/test1_varchar_h2.yml" - }; - - /** - * Constructor which populate table object for each test iteration - * @param table - */ - public JdbcToArrowDataTypesTest(Table table) { - this.table = table; - } - + private static final String BIGINT = "big_int"; + private static final String BINARY = "binary"; + private static final String BIT = "bit"; + private static final String BLOB = "blob"; + private static final String BOOL = "bool"; + private static final String CHAR = "char"; + private static final String CLOB = "clob"; + private static final String DATE = "date"; + private static final String DECIMAL = "decimal"; + private static final String DOUBLE = "double"; + private static final String INT = "int"; + private static final String REAL = "real"; + private static final String SMALLINT = "small_int"; + private static final String TIME = "time"; + private static final String TIMESTAMP = "timestamp"; + private static final String TINYINT = "tiny_int"; + private static final String VARCHAR = "varchar"; + + private static final String[] testFiles = { + "h2/test1_bigint_h2.yml", + "h2/test1_binary_h2.yml", + "h2/test1_bit_h2.yml", + "h2/test1_blob_h2.yml", + "h2/test1_bool_h2.yml", + "h2/test1_char_h2.yml", + "h2/test1_clob_h2.yml", + "h2/test1_date_h2.yml", + "h2/test1_decimal_h2.yml", + "h2/test1_double_h2.yml", + "h2/test1_int_h2.yml", + "h2/test1_real_h2.yml", + "h2/test1_smallint_h2.yml", + "h2/test1_time_h2.yml", + "h2/test1_timestamp_h2.yml", + "h2/test1_tinyint_h2.yml", + "h2/test1_varchar_h2.yml" + }; + + /** + * Constructor which populate table object for each test iteration + * + * @param table + */ + public JdbcToArrowDataTypesTest(Table table) { + this.table = table; + } + /** * This method returns collection of Table object for each test iteration + * * @return * @throws SQLException * @throws ClassNotFoundException * @throws IOException */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.asList(prepareTestData(testFiles, JdbcToArrowDataTypesTest.class)); - } - - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes - */ - @Test - public void testJdbcToArroValues() throws SQLException, IOException { - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(),new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(),new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); - } - - /** - * This method calls the assert methods for various DataSets - * @param root - */ - public void testDataSets(VectorSchemaRoot root) { - switch (table.getType()) { - case BIGINT: - assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case BINARY:case BLOB: - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, table.getBinaryValues()); - break; - case BIT: - assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case BOOL: - assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getBoolValues()); - break; - case CHAR:case VARCHAR: case CLOB: - assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, table.getCharValues()); - break; - case DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case DECIMAL: - assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, table.getBigDecimalValues()); - break; - case DOUBLE: - assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, table.getDoubleValues()); - break; - case INT: - assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case SMALLINT: - assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case TINYINT: - assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); - break; - case REAL: - assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, table.getFloatValues()); - break; - } + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowDataTypesTest.class)); + } + + /** + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes + */ + @Test + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + } + + /** + * This method calls the assert methods for various DataSets + * + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case BIGINT: + assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case BINARY: + case BLOB: + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, table.getBinaryValues()); + break; + case BIT: + assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case BOOL: + assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, table.getBoolValues()); + break; + case CHAR: + case VARCHAR: + case CLOB: + assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, table.getCharValues()); + break; + case DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case DECIMAL: + assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, table.getBigDecimalValues()); + break; + case DOUBLE: + assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, table.getDoubleValues()); + break; + case INT: + assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case SMALLINT: + assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case TINYINT: + assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, table.getIntValues()); + break; + case REAL: + assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, table.getFloatValues()); + break; } + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 4be27d9b46d..7df5278288a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -49,116 +49,120 @@ import org.junit.runners.Parameterized.Parameters; /** - * * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with null values for H2 database - * */ @RunWith(Parameterized.class) public class JdbcToArrowNullTest extends AbstractJdbcToArrowTest { - private static final String NULL = "null"; - private static final String SELECTED_NULL_COLUMN = "selected_null_column"; + private static final String NULL = "null"; + private static final String SELECTED_NULL_COLUMN = "selected_null_column"; - private static final String[] testFiles = { - "h2/test1_all_datatypes_null_h2.yml", - "h2/test1_selected_datatypes_null_h2.yml" - }; - - /** - * Constructor which populate table object for each test iteration - * @param table - */ - public JdbcToArrowNullTest (Table table) { - this.table = table; - } + private static final String[] testFiles = { + "h2/test1_all_datatypes_null_h2.yml", + "h2/test1_selected_datatypes_null_h2.yml" + }; - /** - * This method returns collection of Table object for each test iteration - * @return - * @throws SQLException - * @throws ClassNotFoundException - * @throws IOException - */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.asList(prepareTestData(testFiles, JdbcToArrowNullTest.class)); - } + /** + * Constructor which populate table object for each test iteration + * + * @param table + */ + public JdbcToArrowNullTest(Table table) { + this.table = table; + } - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null values - */ - @Test - public void testJdbcToArroValues() throws SQLException, IOException { - testDataSets(JdbcToArrow.sqlToArrow (conn,table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow (conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()))); - testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow (conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); - } - - - /** - * This method calls the assert methods for various DataSets - * @param root - */ - public void testDataSets(VectorSchemaRoot root) { - switch (table.getType()) { - case NULL: - sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); - break; - case SELECTED_NULL_COLUMN: - sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount()); - break; - } - } - /** - * This method assert tests null values in vectors for all the datatypes - * @param vectors - * @param root - * @param rowCount - */ - public void sqlToArrowTestNullValues(String[] vectors, VectorSchemaRoot root, int rowCount) { - assertNullValues((IntVector)root.getVector(vectors[0]), rowCount); - assertNullValues((BitVector)root.getVector(vectors[1]), rowCount); - assertNullValues((TinyIntVector)root.getVector(vectors[2]), rowCount); - assertNullValues((SmallIntVector)root.getVector(vectors[3]), rowCount); - assertNullValues((BigIntVector)root.getVector(vectors[4]), rowCount); - assertNullValues((DecimalVector)root.getVector(vectors[5]), rowCount); - assertNullValues((Float8Vector)root.getVector(vectors[6]), rowCount); - assertNullValues((Float4Vector)root.getVector(vectors[7]), rowCount); - assertNullValues((TimeMilliVector)root.getVector(vectors[8]), rowCount); - assertNullValues((DateMilliVector)root.getVector(vectors[9]), rowCount); - assertNullValues((TimeStampVector)root.getVector(vectors[10]), rowCount); - assertNullValues((VarBinaryVector)root.getVector(vectors[11]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[12]), rowCount); - assertNullValues((VarBinaryVector)root.getVector(vectors[13]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[14]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[15]), rowCount); - assertNullValues((BitVector)root.getVector(vectors[16]), rowCount); - } - - /** - * This method assert tests null values in vectors for some selected datatypes - * @param vectors - * @param root - * @param rowCount - */ - public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSchemaRoot root, int rowCount) { - assertNullValues((BigIntVector)root.getVector(vectors[0]), rowCount); - assertNullValues((DecimalVector)root.getVector(vectors[1]), rowCount); - assertNullValues((Float8Vector)root.getVector(vectors[2]), rowCount); - assertNullValues((Float4Vector)root.getVector(vectors[3]), rowCount); - assertNullValues((TimeMilliVector)root.getVector(vectors[4]), rowCount); - assertNullValues((DateMilliVector)root.getVector(vectors[5]), rowCount); - assertNullValues((TimeStampVector)root.getVector(vectors[6]), rowCount); - assertNullValues((VarBinaryVector)root.getVector(vectors[7]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[8]), rowCount); - assertNullValues((VarBinaryVector)root.getVector(vectors[9]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[10]), rowCount); - assertNullValues((VarCharVector)root.getVector(vectors[11]), rowCount); - assertNullValues((BitVector)root.getVector(vectors[12]), rowCount); + /** + * This method returns collection of Table object for each test iteration + * + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowNullTest.class)); + } + + /** + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null values + */ + @Test + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + } + + + /** + * This method calls the assert methods for various DataSets + * + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case NULL: + sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); + break; + case SELECTED_NULL_COLUMN: + sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount()); + break; } + } + + /** + * This method assert tests null values in vectors for all the datatypes + * + * @param vectors + * @param root + * @param rowCount + */ + public void sqlToArrowTestNullValues(String[] vectors, VectorSchemaRoot root, int rowCount) { + assertNullValues((IntVector) root.getVector(vectors[0]), rowCount); + assertNullValues((BitVector) root.getVector(vectors[1]), rowCount); + assertNullValues((TinyIntVector) root.getVector(vectors[2]), rowCount); + assertNullValues((SmallIntVector) root.getVector(vectors[3]), rowCount); + assertNullValues((BigIntVector) root.getVector(vectors[4]), rowCount); + assertNullValues((DecimalVector) root.getVector(vectors[5]), rowCount); + assertNullValues((Float8Vector) root.getVector(vectors[6]), rowCount); + assertNullValues((Float4Vector) root.getVector(vectors[7]), rowCount); + assertNullValues((TimeMilliVector) root.getVector(vectors[8]), rowCount); + assertNullValues((DateMilliVector) root.getVector(vectors[9]), rowCount); + assertNullValues((TimeStampVector) root.getVector(vectors[10]), rowCount); + assertNullValues((VarBinaryVector) root.getVector(vectors[11]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[12]), rowCount); + assertNullValues((VarBinaryVector) root.getVector(vectors[13]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[14]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[15]), rowCount); + assertNullValues((BitVector) root.getVector(vectors[16]), rowCount); + } + + /** + * This method assert tests null values in vectors for some selected datatypes + * + * @param vectors + * @param root + * @param rowCount + */ + public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSchemaRoot root, int rowCount) { + assertNullValues((BigIntVector) root.getVector(vectors[0]), rowCount); + assertNullValues((DecimalVector) root.getVector(vectors[1]), rowCount); + assertNullValues((Float8Vector) root.getVector(vectors[2]), rowCount); + assertNullValues((Float4Vector) root.getVector(vectors[3]), rowCount); + assertNullValues((TimeMilliVector) root.getVector(vectors[4]), rowCount); + assertNullValues((DateMilliVector) root.getVector(vectors[5]), rowCount); + assertNullValues((TimeStampVector) root.getVector(vectors[6]), rowCount); + assertNullValues((VarBinaryVector) root.getVector(vectors[7]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[8]), rowCount); + assertNullValues((VarBinaryVector) root.getVector(vectors[9]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[10]), rowCount); + assertNullValues((VarCharVector) root.getVector(vectors[11]), rowCount); + assertNullValues((BitVector) root.getVector(vectors[12]), rowCount); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 8e0606985ac..8de6f3eeaea 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,124 +71,124 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getBinaryValues; /** - * - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types for H2 database * using single test data file - * */ @RunWith(Parameterized.class) public class JdbcToArrowTest extends AbstractJdbcToArrowTest { - - private static final String BIGINT = "BIGINT_FIELD5"; - private static final String BINARY = "BINARY_FIELD12"; - private static final String BIT = "BIT_FIELD17"; - private static final String BLOB = "BLOB_FIELD14"; - private static final String BOOL = "BOOL_FIELD2"; - private static final String CHAR = "CHAR_FIELD16"; - private static final String CLOB = "CLOB_FIELD15"; - private static final String DATE = "DATE_FIELD10"; - private static final String DECIMAL = "DECIMAL_FIELD6"; - private static final String DOUBLE = "DOUBLE_FIELD7"; - private static final String INT = "INT_FIELD1"; - private static final String REAL = "REAL_FIELD8"; - private static final String SMALLINT = "SMALLINT_FIELD4"; - private static final String TIME = "TIME_FIELD9"; - private static final String TIMESTAMP = "TIMESTAMP_FIELD11"; - private static final String TINYINT = "TINYINT_FIELD3"; - private static final String VARCHAR = "VARCHAR_FIELD13"; - - private static final String[] testFiles = {"h2/test1_all_datatypes_h2.yml"}; - - /** - * Constructor which populate table object for each test iteration - * @param table - */ - public JdbcToArrowTest(Table table) { - this.table = table; - } - + + private static final String BIGINT = "BIGINT_FIELD5"; + private static final String BINARY = "BINARY_FIELD12"; + private static final String BIT = "BIT_FIELD17"; + private static final String BLOB = "BLOB_FIELD14"; + private static final String BOOL = "BOOL_FIELD2"; + private static final String CHAR = "CHAR_FIELD16"; + private static final String CLOB = "CLOB_FIELD15"; + private static final String DATE = "DATE_FIELD10"; + private static final String DECIMAL = "DECIMAL_FIELD6"; + private static final String DOUBLE = "DOUBLE_FIELD7"; + private static final String INT = "INT_FIELD1"; + private static final String REAL = "REAL_FIELD8"; + private static final String SMALLINT = "SMALLINT_FIELD4"; + private static final String TIME = "TIME_FIELD9"; + private static final String TIMESTAMP = "TIMESTAMP_FIELD11"; + private static final String TINYINT = "TINYINT_FIELD3"; + private static final String VARCHAR = "VARCHAR_FIELD13"; + + private static final String[] testFiles = {"h2/test1_all_datatypes_h2.yml"}; + + /** + * Constructor which populate table object for each test iteration + * + * @param table + */ + public JdbcToArrowTest(Table table) { + this.table = table; + } + /** * This method returns collection of Table object for each test iteration + * * @return * @throws SQLException * @throws ClassNotFoundException * @throws IOException */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTest.class)); - } - - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one test data file - * - */ - @Test - public void testJdbcToArroValues() throws SQLException, IOException { - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance())); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); - } - - /** - * This method calls the assert methods for various DataSets - * @param root - */ - public void testDataSets(VectorSchemaRoot root) { - assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), - getLongValues(table.getValues(), BIGINT)); - - assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), - getIntValues(table.getValues(), TINYINT)); - - assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), - getIntValues(table.getValues(), SMALLINT)); - - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), - getBinaryValues(table.getValues(), BINARY)); - - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), - getBinaryValues(table.getValues(), BLOB)); - - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), - getCharArray(table.getValues(), CLOB)); - - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), - getCharArray(table.getValues(), VARCHAR)); - - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), - getCharArray(table.getValues(), CHAR)); - - assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), - getIntValues(table.getValues(), INT)); - - assertBitVectorValues((BitVector) root.getVector( BIT), table.getRowCount(), - getIntValues(table.getValues(), BIT)); - - assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), - getBooleanValues(table.getValues(), BOOL)); - - assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), - getLongValues(table.getValues(), DATE)); - - assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), - getLongValues(table.getValues(), TIME)); - - assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), - getLongValues(table.getValues(), TIMESTAMP)); - - assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), - getDecimalValues(table.getValues(), DECIMAL)); - - assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), - getDoubleValues(table.getValues(), DOUBLE)); - - assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), - getFloatValues(table.getValues(), REAL)); - } + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTest.class)); + } + + /** + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one test data file + */ + @Test + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance())); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance())); + } + + /** + * This method calls the assert methods for various DataSets + * + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), + getLongValues(table.getValues(), BIGINT)); + + assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), + getIntValues(table.getValues(), TINYINT)); + + assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), + getIntValues(table.getValues(), SMALLINT)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), + getBinaryValues(table.getValues(), BINARY)); + + assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), + getBinaryValues(table.getValues(), BLOB)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + getCharArray(table.getValues(), CLOB)); + + assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + getCharArray(table.getValues(), VARCHAR)); + + assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + getCharArray(table.getValues(), CHAR)); + + assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), + getIntValues(table.getValues(), INT)); + + assertBitVectorValues((BitVector) root.getVector(BIT), table.getRowCount(), + getIntValues(table.getValues(), BIT)); + + assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), + getBooleanValues(table.getValues(), BOOL)); + + assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), + getLongValues(table.getValues(), DATE)); + + assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), + getLongValues(table.getValues(), TIME)); + + assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), + getLongValues(table.getValues(), TIMESTAMP)); + + assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), + getDecimalValues(table.getValues(), DECIMAL)); + + assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), + getDoubleValues(table.getValues(), DOUBLE)); + + assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), + getFloatValues(table.getValues(), REAL)); + } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 438a0228e7d..87003d001ed 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -27,6 +27,7 @@ import java.util.Calendar; import java.util.Collection; import java.util.TimeZone; + import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.Table; @@ -41,87 +42,94 @@ import org.junit.runners.Parameterized.Parameters; /** - * - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with TimeZone based Date, + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with TimeZone based Date, * Time and Timestamp datatypes for H2 database - * */ @RunWith(Parameterized.class) public class JdbcToArrowTimeZoneTest extends AbstractJdbcToArrowTest { - private static final String EST_DATE = "est_date"; - private static final String EST_TIME = "est_time"; - private static final String EST_TIMESTAMP = "est_timestamp"; - private static final String GMT_DATE = "gmt_date"; - private static final String GMT_TIME = "gmt_time"; - private static final String GMT_TIMESTAMP = "gmt_timestamp"; - private static final String PST_DATE = "pst_date"; - private static final String PST_TIME = "pst_time"; - private static final String PST_TIMESTAMP = "pst_timestamp"; - - private static final String[] testFiles = { - "h2/test1_est_date_h2.yml", - "h2/test1_est_time_h2.yml", - "h2/test1_est_timestamp_h2.yml", - "h2/test1_gmt_date_h2.yml", - "h2/test1_gmt_time_h2.yml", - "h2/test1_gmt_timestamp_h2.yml", - "h2/test1_pst_date_h2.yml", - "h2/test1_pst_time_h2.yml", - "h2/test1_pst_timestamp_h2.yml" - }; + private static final String EST_DATE = "est_date"; + private static final String EST_TIME = "est_time"; + private static final String EST_TIMESTAMP = "est_timestamp"; + private static final String GMT_DATE = "gmt_date"; + private static final String GMT_TIME = "gmt_time"; + private static final String GMT_TIMESTAMP = "gmt_timestamp"; + private static final String PST_DATE = "pst_date"; + private static final String PST_TIME = "pst_time"; + private static final String PST_TIMESTAMP = "pst_timestamp"; - /** - * Constructor which populate table object for each test iteration - * @param table - */ - public JdbcToArrowTimeZoneTest (Table table) { - this.table = table; - } + private static final String[] testFiles = { + "h2/test1_est_date_h2.yml", + "h2/test1_est_time_h2.yml", + "h2/test1_est_timestamp_h2.yml", + "h2/test1_gmt_date_h2.yml", + "h2/test1_gmt_time_h2.yml", + "h2/test1_gmt_timestamp_h2.yml", + "h2/test1_pst_date_h2.yml", + "h2/test1_pst_time_h2.yml", + "h2/test1_pst_timestamp_h2.yml" + }; - /** - * This method returns collection of Table object for each test iteration - * @return - * @throws SQLException - * @throws ClassNotFoundException - * @throws IOException - */ - @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTimeZoneTest.class)); - } - - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with TimeZone based Date, - * Time and Timestamp datatype - */ - @Test - public void testJdbcToArroValues() throws SQLException, IOException { - testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); - testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); - } - - /** - * This method calls the assert methods for various DataSets - * @param root - */ - public void testDataSets(VectorSchemaRoot root) { - switch (table.getType()) { - case EST_DATE: case GMT_DATE: case PST_DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case EST_TIME: case GMT_TIME: case PST_TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - case EST_TIMESTAMP: case GMT_TIMESTAMP: case PST_TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); - break; - } + /** + * Constructor which populate table object for each test iteration + * + * @param table + */ + public JdbcToArrowTimeZoneTest(Table table) { + this.table = table; + } + + /** + * This method returns collection of Table object for each test iteration + * + * @return + * @throws SQLException + * @throws ClassNotFoundException + * @throws IOException + */ + @Parameters + public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTimeZoneTest.class)); + } + + /** + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with TimeZone based Date, + * Time and Timestamp datatype + */ + @Test + public void testJdbcToArroValues() throws SQLException, IOException { + testDataSets(JdbcToArrow.sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + testDataSets(JdbcToArrow.sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())))); + } + + /** + * This method calls the assert methods for various DataSets + * + * @param root + */ + public void testDataSets(VectorSchemaRoot root) { + switch (table.getType()) { + case EST_DATE: + case GMT_DATE: + case PST_DATE: + assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIME: + case GMT_TIME: + case PST_TIME: + assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; + case EST_TIMESTAMP: + case GMT_TIMESTAMP: + case PST_TIMESTAMP: + assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, table.getLongValues()); + break; } + } }