From f3d123484e95dfec4d5d83a57de85696aac2cd46 Mon Sep 17 00:00:00 2001 From: siddharth Date: Sun, 17 Sep 2017 16:16:39 -0700 Subject: [PATCH 1/2] ARROW-1547: Fix 8x memory over-allocation in BitVector --- .../src/main/java/org/apache/arrow/vector/BitVector.java | 2 +- .../apache/arrow/vector/TestBufferOwnershipTransfer.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 8a60273e179..0d442c62bf5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -43,7 +43,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe private final Mutator mutator = new Mutator(); int valueCount; - private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION; + private int allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); private int allocationMonitor = 0; public BitVector(String name, BufferAllocator allocator) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 54fc3067170..1a801a63ec6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -47,7 +47,9 @@ public void testTransferFixedWidth() { v1.makeTransferPair(v2).transfer(); assertEquals(0, childAllocator1.getAllocatedMemory()); - assertEquals(5 * 4096, childAllocator2.getAllocatedMemory()); + int expectedBitVector = 512; + int expectedValueVector = 4096*4; + assertEquals(expectedBitVector + expectedValueVector, childAllocator2.getAllocatedMemory()); } @Test @@ -66,7 +68,10 @@ public void testTransferVariableidth() { v1.makeTransferPair(v2).transfer(); assertEquals(0, childAllocator1.getAllocatedMemory()); - int expected = 8 * 4096 + 4 * 4096 + 4096; + int expectedValueVector = 4096*8; + int expectedOffsetVector = 4096*4; + int expectedBitVector = 512; + int expected = expectedBitVector + expectedOffsetVector + expectedValueVector; assertEquals(expected, childAllocator2.getAllocatedMemory()); } From c92164a0ddbaa72d0e178af1d61170a3a4cddf9c Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 18 Sep 2017 21:11:48 -0700 Subject: [PATCH 2/2] addressed review comments --- .../vector/src/main/java/org/apache/arrow/vector/BitVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 0d442c62bf5..c685ffe12de 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -175,7 +175,7 @@ public boolean allocateNewSafe() { @Override public void reset() { valueCount = 0; - allocationSizeInBytes = INITIAL_VALUE_ALLOCATION; + allocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); allocationMonitor = 0; zeroVector(); super.reset();