From 9db43371c3b60cd65486c135abcd9742bb157b05 Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Thu, 8 Dec 2016 21:24:29 -0800 Subject: [PATCH 1/3] ARROW-402: avoid empty buffers --- .../main/java/org/apache/arrow/vector/file/ArrowReader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java index cd520da54f2..30324cdac0e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java @@ -139,7 +139,10 @@ public ArrowRecordBatch readRecordBatch(ArrowBlock recordBatchBlock) throws IOEx for (int i = 0; i < recordBatchFB.buffersLength(); ++i) { Buffer bufferFB = recordBatchFB.buffers(i); LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", bufferFB.offset(), bufferFB.length())); - ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length()); + ArrowBuf vectorBuffer = + bufferFB.length() == 0 ? + allocator.getEmpty() + : body.slice((int)bufferFB.offset(), (int)bufferFB.length()); buffers.add(vectorBuffer); } ArrowRecordBatch arrowRecordBatch = new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers); From 7968813e2f2920244a133fd0ff8ae524ea6973b1 Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Fri, 9 Dec 2016 08:53:40 -0800 Subject: [PATCH 2/3] actual fix --- .../src/main/java/org/apache/arrow/vector/BitVector.java | 1 - .../main/java/org/apache/arrow/vector/file/ArrowReader.java | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 7ce1236b2ec..072a12aa045 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -53,7 +53,6 @@ public BitVector(String name, BufferAllocator allocator) { public void load(ArrowFieldNode fieldNode, ArrowBuf data) { // When the vector is all nulls or all defined, the content of the buffer can be omitted if (data.readableBytes() == 0 && fieldNode.getLength() != 0) { - data.release(); int count = fieldNode.getLength(); allocateNew(count); int n = getSizeFromCount(count); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java index 30324cdac0e..cd520da54f2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java @@ -139,10 +139,7 @@ public ArrowRecordBatch readRecordBatch(ArrowBlock recordBatchBlock) throws IOEx for (int i = 0; i < recordBatchFB.buffersLength(); ++i) { Buffer bufferFB = recordBatchFB.buffers(i); LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", bufferFB.offset(), bufferFB.length())); - ArrowBuf vectorBuffer = - bufferFB.length() == 0 ? - allocator.getEmpty() - : body.slice((int)bufferFB.offset(), (int)bufferFB.length()); + ArrowBuf vectorBuffer = body.slice((int)bufferFB.offset(), (int)bufferFB.length()); buffers.add(vectorBuffer); } ArrowRecordBatch arrowRecordBatch = new ArrowRecordBatch(recordBatchFB.length(), nodes, buffers); From 3269ca86c6f2dae218a793cb8f83cb8f76f4f1fb Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Fri, 9 Dec 2016 08:58:31 -0800 Subject: [PATCH 3/3] with tests --- .../java/org/apache/arrow/vector/TestVectorUnloadLoad.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 9dfe8d840e4..3455efe245e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -112,7 +112,8 @@ public void testLoadEmptyValidityBuffer() throws IOException { new Field("intNull", true, new ArrowType.Int(32, true), Collections.emptyList()) )); int count = 10; - ArrowBuf validity = allocator.getEmpty(); + // using Allocator.getEmpty() turns off ref counting + ArrowBuf validity = allocator.buffer(10).slice(0, 0); ArrowBuf values = allocator.buffer(count * 4); // integers for (int i = 0; i < count; i++) { values.setInt(i * 4, i); @@ -153,6 +154,7 @@ public void testLoadEmptyValidityBuffer() throws IOException { assertFalse(intDefinedVector.getAccessor().isNull(count + 10)); assertEquals(1234, intDefinedVector.getAccessor().get(count + 10)); } finally { + validity.release(); values.release(); } }