From 12c37d717010e5b683074633b273b0bad0180e05 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Tue, 17 Apr 2018 20:16:12 -0700 Subject: [PATCH] ORC-344. Support the new Decimal64ColumnVector. Fixes #250 Signed-off-by: Owen O'Malley --- .../java/org/apache/orc/TypeDescription.java | 65 ++++++-- .../orc/impl/ConvertTreeReaderFactory.java | 12 +- .../apache/orc/impl/SerializationUtils.java | 12 +- .../apache/orc/impl/TreeReaderFactory.java | 88 +++++++++-- .../orc/impl/writer/DecimalTreeWriter.java | 64 +++++++- .../org/apache/orc/TestVectorOrcFile.java | 141 ++++++++++++++++++ 6 files changed, 335 insertions(+), 47 deletions(-) diff --git a/java/core/src/java/org/apache/orc/TypeDescription.java b/java/core/src/java/org/apache/orc/TypeDescription.java index d7e81cd9cd..264adff20a 100644 --- a/java/core/src/java/org/apache/orc/TypeDescription.java +++ b/java/core/src/java/org/apache/orc/TypeDescription.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector; @@ -45,6 +46,7 @@ public class TypeDescription private static final int MAX_SCALE = 38; private static final int DEFAULT_PRECISION = 38; private static final int DEFAULT_SCALE = 10; + public static final int MAX_DECIMAL64_PRECISION = 18; private static final int DEFAULT_LENGTH = 256; static final Pattern UNQUOTED_NAMES = Pattern.compile("^[a-zA-Z0-9_]+$"); @@ -622,7 +624,7 @@ public int getMaximumId() { return maxId; } - private ColumnVector createColumn(int maxSize) { + private ColumnVector createColumn(RowBatchVersion version, int maxSize) { switch (category) { case BOOLEAN: case BYTE: @@ -637,7 +639,12 @@ private ColumnVector createColumn(int maxSize) { case DOUBLE: return new DoubleColumnVector(maxSize); case DECIMAL: - return new DecimalColumnVector(maxSize, precision, scale); + if (version == RowBatchVersion.ORIGINAL || + precision > MAX_DECIMAL64_PRECISION) { + return new DecimalColumnVector(maxSize, precision, scale); + } else { + return new Decimal64ColumnVector(maxSize, precision, scale); + } case STRING: case BINARY: case CHAR: @@ -646,7 +653,7 @@ private ColumnVector createColumn(int maxSize) { case STRUCT: { ColumnVector[] fieldVector = new ColumnVector[children.size()]; for(int i=0; i < fieldVector.length; ++i) { - fieldVector[i] = children.get(i).createColumn(maxSize); + fieldVector[i] = children.get(i).createColumn(version, maxSize); } return new StructColumnVector(maxSize, fieldVector); @@ -654,40 +661,72 @@ private ColumnVector createColumn(int maxSize) { case UNION: { ColumnVector[] fieldVector = new ColumnVector[children.size()]; for(int i=0; i < fieldVector.length; ++i) { - fieldVector[i] = children.get(i).createColumn(maxSize); + fieldVector[i] = children.get(i).createColumn(version, maxSize); } return new UnionColumnVector(maxSize, fieldVector); } case LIST: return new ListColumnVector(maxSize, - children.get(0).createColumn(maxSize)); + children.get(0).createColumn(version, maxSize)); case MAP: return new MapColumnVector(maxSize, - children.get(0).createColumn(maxSize), - children.get(1).createColumn(maxSize)); + children.get(0).createColumn(version, maxSize), + children.get(1).createColumn(version, maxSize)); default: throw new IllegalArgumentException("Unknown type " + category); } } - public VectorizedRowBatch createRowBatch(int maxSize) { + /** + * Specify the version of the VectorizedRowBatch that the user desires. + */ + enum RowBatchVersion { + ORIGINAL, + USE_DECIMAL64; + } + + VectorizedRowBatch createRowBatch(RowBatchVersion version, int size) { VectorizedRowBatch result; if (category == Category.STRUCT) { - result = new VectorizedRowBatch(children.size(), maxSize); + result = new VectorizedRowBatch(children.size(), size); for(int i=0; i < result.cols.length; ++i) { - result.cols[i] = children.get(i).createColumn(maxSize); + result.cols[i] = children.get(i).createColumn(version, size); } } else { - result = new VectorizedRowBatch(1, maxSize); - result.cols[0] = createColumn(maxSize); + result = new VectorizedRowBatch(1, size); + result.cols[0] = createColumn(version, size); } result.reset(); return result; } + /** + * Create a VectorizedRowBatch that uses Decimal64ColumnVector for + * short (p <= 18) decimals. + * @return a new VectorizedRowBatch + */ + public VectorizedRowBatch createRowBatchV2() { + return createRowBatch(RowBatchVersion.USE_DECIMAL64, + VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Create a VectorizedRowBatch with the original ColumnVector types + * @param maxSize the maximum size of the batch + * @return a new VectorizedRowBatch + */ + public VectorizedRowBatch createRowBatch(int maxSize) { + return createRowBatch(RowBatchVersion.ORIGINAL, maxSize); + } + + /** + * Create a VectorizedRowBatch with the original ColumnVector types + * @return a new VectorizedRowBatch + */ public VectorizedRowBatch createRowBatch() { - return createRowBatch(VectorizedRowBatch.DEFAULT_SIZE); + return createRowBatch(RowBatchVersion.ORIGINAL, + VectorizedRowBatch.DEFAULT_SIZE); } /** diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java index 7964340a85..eb731fa80d 100644 --- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java +++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java @@ -595,7 +595,7 @@ public static class AnyIntegerFromDecimalTreeReader extends ConvertTreeReader { this.precision = fileType.getPrecision(); this.scale = fileType.getScale(); this.readerType = readerType; - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, precision, scale, context); setConvertTreeReader(decimalTreeReader); } @@ -848,7 +848,7 @@ public static class FloatFromDecimalTreeReader extends ConvertTreeReader { super(columnId); this.precision = fileType.getPrecision(); this.scale = fileType.getScale(); - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, precision, scale, context); setConvertTreeReader(decimalTreeReader); } @@ -1054,7 +1054,7 @@ public static class DoubleFromDecimalTreeReader extends ConvertTreeReader { super(columnId); this.precision = fileType.getPrecision(); this.scale = fileType.getScale(); - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, precision, scale, context); setConvertTreeReader(decimalTreeReader); } @@ -1387,7 +1387,7 @@ public static class DecimalFromDecimalTreeReader extends ConvertTreeReader { super(columnId); filePrecision = fileType.getPrecision(); fileScale = fileType.getScale(); - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, filePrecision, fileScale, context); setConvertTreeReader(decimalTreeReader); } @@ -1565,7 +1565,7 @@ public static class StringGroupFromDecimalTreeReader extends ConvertTreeReader { this.precision = fileType.getPrecision(); this.scale = fileType.getScale(); this.readerType = readerType; - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, precision, scale, context); setConvertTreeReader(decimalTreeReader); scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; } @@ -1904,7 +1904,7 @@ public static class TimestampFromDecimalTreeReader extends ConvertTreeReader { super(columnId); this.precision = fileType.getPrecision(); this.scale = fileType.getScale(); - decimalTreeReader = new DecimalTreeReader(columnId, context); + decimalTreeReader = new DecimalTreeReader(columnId, precision, scale, context); setConvertTreeReader(decimalTreeReader); } diff --git a/java/core/src/java/org/apache/orc/impl/SerializationUtils.java b/java/core/src/java/org/apache/orc/impl/SerializationUtils.java index 5c05adea69..3bed475af2 100644 --- a/java/core/src/java/org/apache/orc/impl/SerializationUtils.java +++ b/java/core/src/java/org/apache/orc/impl/SerializationUtils.java @@ -38,15 +38,17 @@ public SerializationUtils() { public void writeVulong(OutputStream output, long value) throws IOException { + int posn = 0; while (true) { if ((value & ~0x7f) == 0) { - output.write((byte) value); - return; + writeBuffer[posn++] = (byte) value; + break; } else { - output.write((byte) (0x80 | (value & 0x7f))); + writeBuffer[posn++] = (byte)(0x80 | (value & 0x7f)); value >>>= 7; } } + output.write(writeBuffer, 0, posn); } public void writeVslong(OutputStream output, @@ -55,7 +57,7 @@ public void writeVslong(OutputStream output, } - public long readVulong(InputStream in) throws IOException { + public static long readVulong(InputStream in) throws IOException { long result = 0; long b; int offset = 0; @@ -70,7 +72,7 @@ public long readVulong(InputStream in) throws IOException { return result; } - public long readVslong(InputStream in) throws IOException { + public static long readVslong(InputStream in) throws IOException { long result = readVulong(in); return (result >>> 1) ^ -(result & 1); } diff --git a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java index b33ad111f4..652067d920 100644 --- a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java +++ b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector; @@ -1096,19 +1097,31 @@ void skipRows(long items) throws IOException { } public static class DecimalTreeReader extends TreeReader { + protected final int precision; + protected final int scale; protected InStream valueStream; protected IntegerReader scaleReader = null; private int[] scratchScaleVector; private byte[] scratchBytes; - DecimalTreeReader(int columnId, Context context) throws IOException { - this(columnId, null, null, null, null, context); - } - - protected DecimalTreeReader(int columnId, InStream present, - InStream valueStream, InStream scaleStream, OrcProto.ColumnEncoding encoding, Context context) - throws IOException { + DecimalTreeReader(int columnId, + int precision, + int scale, + Context context) throws IOException { + this(columnId, null, null, null, null, precision, scale, context); + } + + protected DecimalTreeReader(int columnId, + InStream present, + InStream valueStream, + InStream scaleStream, + OrcProto.ColumnEncoding encoding, + int precision, + int scale, + Context context) throws IOException { super(columnId, present, context); + this.precision = precision; + this.scale = scale; this.scratchScaleVector = new int[VectorizedRowBatch.DEFAULT_SIZE]; this.valueStream = valueStream; this.scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ]; @@ -1150,14 +1163,9 @@ public void seek(PositionProvider index) throws IOException { scaleReader.seek(index); } - @Override - public void nextVector(ColumnVector previousVector, - boolean[] isNull, - final int batchSize) throws IOException { - final DecimalColumnVector result = (DecimalColumnVector) previousVector; - // Read present/isNull stream - super.nextVector(result, isNull, batchSize); - + private void nextVector(DecimalColumnVector result, + boolean[] isNull, + final int batchSize) throws IOException { if (batchSize > scratchScaleVector.length) { scratchScaleVector = new int[(int) batchSize]; } @@ -1193,6 +1201,53 @@ public void nextVector(ColumnVector previousVector, } } + private void nextVector(Decimal64ColumnVector result, + boolean[] isNull, + final int batchSize) throws IOException { + if (precision > TypeDescription.MAX_DECIMAL64_PRECISION) { + throw new IllegalArgumentException("Reading large precision type into" + + " Decimal64ColumnVector."); + } + + if (batchSize > scratchScaleVector.length) { + scratchScaleVector = new int[(int) batchSize]; + } + // read the scales + scaleReader.nextVector(result, scratchScaleVector, batchSize); + if (result.noNulls) { + for (int r=0; r < batchSize; ++r) { + result.vector[r] = SerializationUtils.readVslong(valueStream); + for(int s=scratchScaleVector[r]; s < scale; ++s) { + result.vector[r] *= 10; + } + } + } else if (!result.isRepeating || !result.isNull[0]) { + for (int r=0; r < batchSize; ++r) { + if (!result.isNull[r]) { + result.vector[r] = SerializationUtils.readVslong(valueStream); + for(int s=scratchScaleVector[r]; s < scale; ++s) { + result.vector[r] *= 10; + } + } + } + } + result.precision = (short) precision; + result.scale = (short) scale; + } + + @Override + public void nextVector(ColumnVector result, + boolean[] isNull, + final int batchSize) throws IOException { + // Read present/isNull stream + super.nextVector(result, isNull, batchSize); + if (result instanceof Decimal64ColumnVector) { + nextVector((Decimal64ColumnVector) result, isNull, batchSize); + } else { + nextVector((DecimalColumnVector) result, isNull, batchSize); + } + } + @Override void skipRows(long items) throws IOException { items = countNonNulls(items); @@ -2186,7 +2241,8 @@ public static TreeReader createTreeReader(TypeDescription readerType, case DATE: return new DateTreeReader(fileType.getId(), context); case DECIMAL: - return new DecimalTreeReader(fileType.getId(), context); + return new DecimalTreeReader(fileType.getId(), fileType.getPrecision(), + fileType.getScale(), context); case STRUCT: return new StructTreeReader(fileType.getId(), readerType, context); case LIST: diff --git a/java/core/src/java/org/apache/orc/impl/writer/DecimalTreeWriter.java b/java/core/src/java/org/apache/orc/impl/writer/DecimalTreeWriter.java index 0428253729..5d88372daf 100644 --- a/java/core/src/java/org/apache/orc/impl/writer/DecimalTreeWriter.java +++ b/java/core/src/java/org/apache/orc/impl/writer/DecimalTreeWriter.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -28,11 +29,13 @@ import org.apache.orc.impl.IntegerWriter; import org.apache.orc.impl.PositionRecorder; import org.apache.orc.impl.PositionedOutputStream; +import org.apache.orc.impl.SerializationUtils; import java.io.IOException; public class DecimalTreeWriter extends TreeWriterBase { private final PositionedOutputStream valueStream; + private final SerializationUtils utils = new SerializationUtils(); // These scratch buffers allow us to serialize decimals much faster. private final long[] scratchLongs; @@ -68,14 +71,11 @@ OrcProto.ColumnEncoding.Builder getEncoding() { return result; } - @Override - public void writeBatch(ColumnVector vector, int offset, + private void writeBatch(DecimalColumnVector vector, int offset, int length) throws IOException { - super.writeBatch(vector, offset, length); - DecimalColumnVector vec = (DecimalColumnVector) vector; if (vector.isRepeating) { if (vector.noNulls || !vector.isNull[0]) { - HiveDecimalWritable value = vec.vector[0]; + HiveDecimalWritable value = vector.vector[0]; indexStatistics.updateDecimal(value); if (createBloomFilter) { String str = value.toString(scratchBuffer); @@ -92,8 +92,8 @@ public void writeBatch(ColumnVector vector, int offset, } } else { for (int i = 0; i < length; ++i) { - if (vec.noNulls || !vec.isNull[i + offset]) { - HiveDecimalWritable value = vec.vector[i + offset]; + if (vector.noNulls || !vector.isNull[i + offset]) { + HiveDecimalWritable value = vector.vector[i + offset]; value.serializationUtilsWrite(valueStream, scratchLongs); scaleStream.write(value.scale()); indexStatistics.updateDecimal(value); @@ -109,6 +109,56 @@ public void writeBatch(ColumnVector vector, int offset, } } + private void writeBatch(Decimal64ColumnVector vector, int offset, + int length) throws IOException { + if (vector.isRepeating) { + if (vector.noNulls || !vector.isNull[0]) { + HiveDecimalWritable value = vector.getScratchWritable(); + value.setFromLongAndScale(vector.vector[0], vector.scale); + indexStatistics.updateDecimal(value); + if (createBloomFilter) { + String str = value.toString(scratchBuffer); + if (bloomFilter != null) { + bloomFilter.addString(str); + } + bloomFilterUtf8.addString(str); + } + for (int i = 0; i < length; ++i) { + utils.writeVslong(valueStream, vector.vector[0]); + scaleStream.write(vector.scale); + } + } + } else { + HiveDecimalWritable value = vector.getScratchWritable(); + for (int i = 0; i < length; ++i) { + if (vector.noNulls || !vector.isNull[i + offset]) { + utils.writeVslong(valueStream, vector.vector[i + offset]); + scaleStream.write(vector.scale); + value.setFromLongAndScale(vector.vector[i + offset], vector.scale); + indexStatistics.updateDecimal(value); + if (createBloomFilter) { + String str = value.toString(scratchBuffer); + if (bloomFilter != null) { + bloomFilter.addString(str); + } + bloomFilterUtf8.addString(str); + } + } + } + } + } + + @Override + public void writeBatch(ColumnVector vector, int offset, + int length) throws IOException { + super.writeBatch(vector, offset, length); + if (vector instanceof Decimal64ColumnVector) { + writeBatch((Decimal64ColumnVector) vector, offset, length); + } else { + writeBatch((DecimalColumnVector) vector, offset, length); + } + } + @Override public void writeStripe(OrcProto.StripeFooter.Builder builder, OrcProto.StripeStatistics.Builder stats, diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java index b254fb163d..f8ed256d2e 100644 --- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java +++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java @@ -18,6 +18,7 @@ package org.apache.orc; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; import org.apache.orc.impl.OrcCodecPool; import org.apache.orc.impl.WriterImpl; @@ -1467,6 +1468,146 @@ private static void setUnion(VectorizedRowBatch batch, int rowId, } } + /** + * Test writing with the new decimal and reading with the new and old. + */ + @Test + public void testDecimal64Writing() throws Exception { + TypeDescription schema = TypeDescription.fromString("struct"); + VectorizedRowBatch batch = schema.createRowBatchV2(); + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf) + .setSchema(schema) + .compress(CompressionKind.NONE)); + Decimal64ColumnVector cv = (Decimal64ColumnVector) batch.cols[0]; + cv.precision = 18; + cv.scale = 3; + cv.vector[0] = 1; + for(int r=1; r < 18; r++) { + cv.vector[r] = cv.vector[r-1] * 10; + } + cv.vector[18] = -2000; + batch.size = 19; + writer.addRowBatch(batch); + writer.close(); + + Reader reader = OrcFile.createReader(testFilePath, + OrcFile.readerOptions(conf).filesystem(fs)); + RecordReader rows = reader.rows(); + batch = schema.createRowBatchV2(); + cv = (Decimal64ColumnVector) batch.cols[0]; + assertTrue(rows.nextBatch(batch)); + assertEquals(19, batch.size); + assertEquals(18, cv.precision); + assertEquals(3, cv.scale); + assertEquals("row 0", 1, cv.vector[0]); + for(int r=1; r < 18; ++r) { + assertEquals("row " + r, 10 * cv.vector[r-1], cv.vector[r]); + } + assertEquals(-2000, cv.vector[18]); + assertFalse(rows.nextBatch(batch)); + + // test with old batch + rows = reader.rows(); + batch = schema.createRowBatch(); + DecimalColumnVector oldCv = (DecimalColumnVector) batch.cols[0]; + assertTrue(rows.nextBatch(batch)); + assertEquals(19, batch.size); + assertEquals(18, oldCv.precision); + assertEquals(3, oldCv.scale); + assertEquals("0.001", oldCv.vector[0].toString()); + assertEquals("0.01", oldCv.vector[1].toString()); + assertEquals("0.1", oldCv.vector[2].toString()); + assertEquals("1", oldCv.vector[3].toString()); + assertEquals("10", oldCv.vector[4].toString()); + assertEquals("100", oldCv.vector[5].toString()); + assertEquals("1000", oldCv.vector[6].toString()); + assertEquals("10000", oldCv.vector[7].toString()); + assertEquals("100000", oldCv.vector[8].toString()); + assertEquals("1000000", oldCv.vector[9].toString()); + assertEquals("10000000", oldCv.vector[10].toString()); + assertEquals("100000000", oldCv.vector[11].toString()); + assertEquals("1000000000", oldCv.vector[12].toString()); + assertEquals("10000000000", oldCv.vector[13].toString()); + assertEquals("100000000000", oldCv.vector[14].toString()); + assertEquals("1000000000000", oldCv.vector[15].toString()); + assertEquals("10000000000000", oldCv.vector[16].toString()); + assertEquals("100000000000000", oldCv.vector[17].toString()); + assertEquals("-2", oldCv.vector[18].toString()); + assertFalse(rows.nextBatch(batch)); + } + + /** + * Test writing with the old decimal and reading with the new and old. + */ + @Test + public void testDecimal64Reading() throws Exception { + TypeDescription schema = TypeDescription.fromString("struct"); + VectorizedRowBatch batch = schema.createRowBatch(); + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf) + .setSchema(schema) + .compress(CompressionKind.NONE)); + DecimalColumnVector cv = (DecimalColumnVector) batch.cols[0]; + cv.precision = 18; + cv.scale = 3; + long base = 1; + for(int r=0; r < 18; r++) { + cv.vector[r].setFromLongAndScale(base, 4); + base *= 10; + } + cv.vector[18].setFromLong(-2); + batch.size = 19; + writer.addRowBatch(batch); + writer.close(); + + // test with new batch + Reader reader = OrcFile.createReader(testFilePath, + OrcFile.readerOptions(conf).filesystem(fs)); + RecordReader rows = reader.rows(); + batch = schema.createRowBatchV2(); + Decimal64ColumnVector newCv = (Decimal64ColumnVector) batch.cols[0]; + assertTrue(rows.nextBatch(batch)); + assertEquals(19, batch.size); + assertEquals(18, newCv.precision); + assertEquals(4, newCv.scale); + assertEquals("row 0", 1, newCv.vector[0]); + for(int r=1; r < 18; ++r) { + assertEquals("row " + r, 10 * newCv.vector[r-1], newCv.vector[r]); + } + assertEquals(-20000, newCv.vector[18]); + assertFalse(rows.nextBatch(batch)); + + // test with old batch + rows = reader.rows(); + batch = schema.createRowBatch(); + cv = (DecimalColumnVector) batch.cols[0]; + assertTrue(rows.nextBatch(batch)); + assertEquals(19, batch.size); + assertEquals(18, cv.precision); + assertEquals(4, cv.scale); + assertEquals("0.0001", cv.vector[0].toString()); + assertEquals("0.001", cv.vector[1].toString()); + assertEquals("0.01", cv.vector[2].toString()); + assertEquals("0.1", cv.vector[3].toString()); + assertEquals("1", cv.vector[4].toString()); + assertEquals("10", cv.vector[5].toString()); + assertEquals("100", cv.vector[6].toString()); + assertEquals("1000", cv.vector[7].toString()); + assertEquals("10000", cv.vector[8].toString()); + assertEquals("100000", cv.vector[9].toString()); + assertEquals("1000000", cv.vector[10].toString()); + assertEquals("10000000", cv.vector[11].toString()); + assertEquals("100000000", cv.vector[12].toString()); + assertEquals("1000000000", cv.vector[13].toString()); + assertEquals("10000000000", cv.vector[14].toString()); + assertEquals("100000000000", cv.vector[15].toString()); + assertEquals("1000000000000", cv.vector[16].toString()); + assertEquals("10000000000000", cv.vector[17].toString()); + assertEquals("-2", cv.vector[18].toString()); + assertFalse(rows.nextBatch(batch)); + } + /** * We test union, timestamp, and decimal separately since we need to make the * object inspector manually. (The Hive reflection-based doesn't handle