From bacf9a7ea699702eb126e97b3a3b931259131557 Mon Sep 17 00:00:00 2001 From: Yurui Zhou Date: Thu, 25 Apr 2019 14:47:42 +0800 Subject: [PATCH] add hasNull flag and remove unnecessary checks --- .../arrow/vector/BaseFixedWidthVector.java | 45 ++++++++++++++++++- .../arrow/vector/BaseVariableWidthVector.java | 22 ++++++++- .../org/apache/arrow/vector/BigIntVector.java | 16 +------ .../org/apache/arrow/vector/BitVector.java | 15 +------ .../apache/arrow/vector/DateDayVector.java | 14 +----- .../apache/arrow/vector/DateMilliVector.java | 15 +------ .../apache/arrow/vector/DecimalVector.java | 14 +----- .../arrow/vector/FixedSizeBinaryVector.java | 7 +-- .../org/apache/arrow/vector/Float4Vector.java | 14 +----- .../org/apache/arrow/vector/Float8Vector.java | 14 +----- .../org/apache/arrow/vector/IntVector.java | 14 +----- .../arrow/vector/IntervalDayVector.java | 14 +----- .../arrow/vector/IntervalYearVector.java | 14 +----- .../apache/arrow/vector/SmallIntVector.java | 14 +----- .../apache/arrow/vector/TimeMicroVector.java | 14 +----- .../apache/arrow/vector/TimeMilliVector.java | 14 +----- .../apache/arrow/vector/TimeNanoVector.java | 14 +----- .../apache/arrow/vector/TimeSecVector.java | 14 +----- .../arrow/vector/TimeStampMicroTZVector.java | 2 +- .../arrow/vector/TimeStampMicroVector.java | 2 +- .../arrow/vector/TimeStampMilliTZVector.java | 2 +- .../arrow/vector/TimeStampMilliVector.java | 2 +- .../arrow/vector/TimeStampNanoTZVector.java | 2 +- .../arrow/vector/TimeStampNanoVector.java | 2 +- .../arrow/vector/TimeStampSecTZVector.java | 2 +- .../arrow/vector/TimeStampSecVector.java | 2 +- .../apache/arrow/vector/TimeStampVector.java | 14 +----- .../apache/arrow/vector/TinyIntVector.java | 14 +----- .../org/apache/arrow/vector/UInt1Vector.java | 14 +----- .../org/apache/arrow/vector/UInt2Vector.java | 14 +----- .../org/apache/arrow/vector/UInt4Vector.java | 14 +----- .../org/apache/arrow/vector/UInt8Vector.java | 14 +----- .../apache/arrow/vector/VarBinaryVector.java | 6 +++ .../apache/arrow/vector/VarCharVector.java | 6 +++ .../vector/complex/FixedSizeListVector.java | 23 +++++++++- .../arrow/vector/complex/ListVector.java | 21 ++++++++- 36 files changed, 152 insertions(+), 292 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index fd1055dc687..c0b0fe05f6e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -49,6 +49,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; protected int valueCount; + protected boolean hasNull; public BaseFixedWidthVector(final String name, final BufferAllocator allocator, FieldType fieldType, final int typeWidth) { @@ -60,6 +61,7 @@ public BaseFixedWidthVector(final String name, final BufferAllocator allocator, validityBuffer = allocator.getEmpty(); valueBuffer = allocator.getEmpty(); lastValueCapacity = INITIAL_VALUE_ALLOCATION; + hasNull = true; } @@ -558,6 +560,7 @@ public void transferTo(BaseFixedWidthVector target) { target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; target.valueCount = valueCount; + target.hasNull = hasNull; clear(); } @@ -600,6 +603,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, int offset = startIndex % 8; if (length > 0) { + target.hasNull = hasNull; if (offset == 0) { /* slice */ if (target.validityBuffer != null) { @@ -665,7 +669,42 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, */ @Override public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); + if (hasNull) { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + return 0; + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); + setNullUnsafe(index); + } + + + /** + * Set the element at the given index to null without boundary check. + * + * @param index position of element + */ + public void setNullUnsafe(int index) { + // not really needed to set the bit to 0 as long as + // the buffer always starts from 0. + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setHasNull(true); + } + + /** + * Set the flag to indicate if the vector has null. + * + * @param hasNull flag to indicate if the vector has null + */ + public void setHasNull(boolean hasNull) { + this.hasNull = hasNull; } /** @@ -756,6 +795,10 @@ public boolean isNull(int index) { * @return 1 if element at given index is not null, 0 otherwise */ public int isSet(int index) { + if (!hasNull) { + return 1; + } + final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); final int bitIndex = index & 7; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 9d765a468b4..4333ce4e393 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -50,6 +50,7 @@ public abstract class BaseVariableWidthVector extends BaseValueVector protected int valueCount; protected int lastSet; protected final Field field; + protected boolean hasNull; public BaseVariableWidthVector(final String name, final BufferAllocator allocator, FieldType fieldType) { @@ -63,6 +64,7 @@ public BaseVariableWidthVector(final String name, final BufferAllocator allocato offsetBuffer = allocator.getEmpty(); validityBuffer = allocator.getEmpty(); valueBuffer = allocator.getEmpty(); + hasNull = true; } /* TODO: @@ -672,6 +674,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { public void transferTo(BaseVariableWidthVector target) { compareTypes(target, "transferTo"); target.clear(); + target.hasNull = hasNull; target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; @@ -733,6 +736,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, int offset = startIndex % 8; if (length > 0) { + target.hasNull = hasNull; if (offset == 0) { // slice if (target.validityBuffer != null) { @@ -794,7 +798,10 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * @return the number of null elements. */ public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); + if (hasNull) { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + return 0; } /** @@ -818,6 +825,15 @@ public boolean isNull(int index) { return (isSet(index) == 0); } + /** + * Set the flag to indicate if the vector has null. + * + * @param hasNull flag to indicate if the vector has null + */ + public void setHasNull(boolean hasNull) { + this.hasNull = hasNull; + } + /** * Same as {@link #isNull(int)}. * @@ -825,6 +841,10 @@ public boolean isNull(int index) { * @return 1 if element at given index is not null, 0 otherwise */ public int isSet(int index) { + if (!hasNull) { + return 1; + } + final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); final int bitIndex = index & 7; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java index fb8b24179da..6e4c90468f7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java @@ -197,7 +197,7 @@ public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentEx BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } @@ -251,18 +251,6 @@ public void setSafe(int index, BigIntHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -274,7 +262,7 @@ public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index b806ec7205b..3cd39f7a5d8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -144,6 +144,7 @@ public int getBufferSize() { public void splitAndTransferTo(int startIndex, int length, BaseFixedWidthVector target) { compareTypes(target, "splitAndTransferTo"); target.clear(); + target.hasNull = hasNull; target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, validityBuffer, target.validityBuffer); target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, @@ -401,18 +402,6 @@ public void setSafe(int index, BitHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -425,7 +414,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index 2e92ff13634..36a3c63f00e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -252,18 +252,6 @@ public void setSafe(int index, DateDayHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -276,7 +264,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java index fdd832c03b1..8ba0ac7ae54 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java @@ -203,6 +203,7 @@ public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumen setValue(index, holder.value); } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setHasNull(true); } } @@ -256,18 +257,6 @@ public void setSafe(int index, DateMilliHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -280,7 +269,7 @@ public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 1045b2305ed..31d1e32b6a4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -438,18 +438,6 @@ public void setSafe(int index, DecimalHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -463,7 +451,7 @@ public void set(int index, int isSet, int start, ArrowBuf buffer) { if (isSet > 0) { set(index, start, buffer); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java index 8221fe419d3..be920d95930 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedSizeBinaryVector.java @@ -300,7 +300,7 @@ public void set(int index, NullableFixedSizeBinaryHolder holder) { } else if (holder.isSet > 0) { set(index, holder.buffer); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } @@ -317,11 +317,6 @@ public void setSafe(int index, NullableFixedSizeBinaryHolder holder) { set(index, holder); } - public void setNull(int index) { - handleSafe(index); - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java index c20d500b9f7..f9d1e619f3e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java @@ -253,18 +253,6 @@ public void setSafe(int index, Float4Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, float value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java index fd7e3db572b..81d069255de 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java @@ -253,18 +253,6 @@ public void setSafe(int index, Float8Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, double value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java index 9c585f5e5cd..9bcb09d5baf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java @@ -253,18 +253,6 @@ public void setSafe(int index, IntHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index 2dcc9860d41..c960d2eb1f3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -324,18 +324,6 @@ public void setSafe(int index, IntervalDayHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -349,7 +337,7 @@ public void set(int index, int isSet, int days, int milliseconds) { if (isSet > 0) { set(index, days, milliseconds); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 1de643bec0b..db9cfccc2c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -286,18 +286,6 @@ public void setSafe(int index, IntervalYearHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -310,7 +298,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java index 8e9eb4476a4..97f9b0a1d39 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java @@ -281,18 +281,6 @@ public void setSafe(int index, SmallIntHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -305,7 +293,7 @@ public void set(int index, int isSet, short value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java index 6d9b4fc8b23..391e0d5a5aa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java @@ -253,18 +253,6 @@ public void setSafe(int index, TimeMicroHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java index d7aa9273791..584fbe4076c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java @@ -256,18 +256,6 @@ public void setSafe(int index, TimeMilliHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -280,7 +268,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java index 50e7d8e0e8b..f1009c863e0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java @@ -253,18 +253,6 @@ public void setSafe(int index, TimeNanoHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java index df444e48979..9f11d122c96 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java @@ -253,18 +253,6 @@ public void setSafe(int index, TimeSecHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -277,7 +265,7 @@ public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index 7027f4f1aca..8d459b048b8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampMicroTZHolder holder) throws Illegal BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java index ea71fb7c92c..242057083f4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalAr BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index 7cb83bed7ea..3f4bdc0b61c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampMilliTZHolder holder) throws Illegal BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java index b05749e8951..4fd316da146 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java @@ -144,7 +144,7 @@ public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalAr BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index eacc891358b..209b7df11d3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalA BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java index ccc17de5c93..f4a163eb4cd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java @@ -144,7 +144,7 @@ public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArg BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index 3f24c18de03..b337f988aea 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalAr BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java index 2293c10ab92..f9e8db7cdc0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java @@ -145,7 +145,7 @@ public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgu BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java index a84a96266a0..30c0ddae809 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampVector.java @@ -127,18 +127,6 @@ public void setSafe(int index, long value) { set(index, value); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -151,7 +139,7 @@ public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java index d5c0beff499..c583ed70d9d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java @@ -280,18 +280,6 @@ public void setSafe(int index, TinyIntHolder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - /** * Store the given value at a particular position in the vector. isSet indicates * whether the value is NULL or not. @@ -304,7 +292,7 @@ public void set(int index, int isSet, byte value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java index 8b19cdf8580..982878861d2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java @@ -236,23 +236,11 @@ public void setSafe(int index, UInt1Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - public void set(int index, int isSet, byte value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java index 5e1237e02e6..2f826ae0ef4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java @@ -236,23 +236,11 @@ public void setSafe(int index, UInt2Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - public void set(int index, int isSet, char value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java index edaef22ef3d..e55e84ffbfe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java @@ -208,23 +208,11 @@ public void setSafe(int index, UInt4Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - public void set(int index, int isSet, int value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java index cab162b55c1..2eb99fe3d7c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java @@ -208,23 +208,11 @@ public void setSafe(int index, UInt8Holder holder) { set(index, holder); } - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index) { - handleSafe(index); - // not really needed to set the bit to 0 as long as - // the buffer always starts from 0. - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - public void set(int index, int isSet, long value) { if (isSet > 0) { set(index, value); } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setNullUnsafe(index); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index a0f4e7b0c1f..c43a749062f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -240,6 +240,9 @@ public void set(int index, NullableVarBinaryHolder holder) { assert index >= 0; fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + if (holder.isSet == 0) { + setHasNull(true); + } final int dataLength = holder.end - holder.start; final int startOffset = getstartOffset(index); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); @@ -261,6 +264,9 @@ public void setSafe(int index, NullableVarBinaryHolder holder) { fillEmpties(index); handleSafe(index, dataLength); BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + if (holder.isSet == 0) { + setHasNull(true); + } final int startOffset = getstartOffset(index); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index a6a23eaf77f..e8ca25cf331 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -239,6 +239,9 @@ public void set(int index, NullableVarCharHolder holder) { assert index >= 0; fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + if (holder.isSet == 0) { + setHasNull(true); + } final int dataLength = holder.end - holder.start; final int startOffset = getstartOffset(index); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); @@ -260,6 +263,9 @@ public void setSafe(int index, NullableVarCharHolder holder) { fillEmpties(index); handleSafe(index, dataLength); BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + if (holder.isSet == 0) { + setHasNull(true); + } final int startOffset = getstartOffset(index); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 31bdf14f4ba..2dd4cb64850 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -68,6 +68,7 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a private UnionFixedSizeListReader reader; private int valueCount; private int validityAllocationSizeInBytes; + private boolean hasNull; // deprecated, use FieldType or static constructor instead @Deprecated @@ -92,6 +93,7 @@ public FixedSizeListVector(String name, this.reader = new UnionFixedSizeListReader(this); this.valueCount = 0; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.hasNull = true; } @Override @@ -404,6 +406,10 @@ public boolean isNull(int index) { } public int isSet(int index) { + if (!hasNull) { + return 1; + } + final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); final int bitIndex = index & 7; @@ -412,7 +418,10 @@ public int isSet(int index) { @Override public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); + if (hasNull) { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + return 0; } @Override @@ -429,6 +438,7 @@ public void setNull(int index) { reallocValidityBuffer(); } BitVectorHelper.setValidityBit(validityBuffer, index, 0); + setHasNull(true); } public void setNotNull(int index) { @@ -438,6 +448,15 @@ public void setNotNull(int index) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); } + /** + * Set the flag to indicate if the vector has null. + * + * @param hasNull flag to indicate if the vector has null + */ + public void setHasNull(boolean hasNull) { + this.hasNull = hasNull; + } + @Override public void setValueCount(int valueCount) { this.valueCount = valueCount; @@ -483,6 +502,7 @@ public void transfer() { dataPair.transfer(); to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.setValueCount(valueCount); + to.hasNull = hasNull; clear(); } @@ -490,6 +510,7 @@ public void transfer() { public void splitAndTransfer(int startIndex, int length) { to.clear(); to.allocateNew(); + to.hasNull = hasNull; for (int i = 0; i < length; i++) { copyValueSafe(startIndex + i, i); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 7d98f71477d..18945b6bda2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -64,6 +64,7 @@ public static ListVector empty(String name, BufferAllocator allocator) { private final FieldType fieldType; private int validityAllocationSizeInBytes; private int lastSet; + private boolean hasNull; // deprecated, use FieldType or static constructor instead @Deprecated @@ -85,6 +86,7 @@ public ListVector(String name, BufferAllocator allocator, FieldType fieldType, C this.callBack = callBack; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); this.lastSet = 0; + this.hasNull = true; } @Override @@ -417,6 +419,7 @@ public void transfer() { to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; + to.hasNull = hasNull; if (valueCount > 0) { to.setValueCount(valueCount); } @@ -459,6 +462,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect int offset = startIndex % 8; if (length > 0) { + target.hasNull = hasNull; if (offset == 0) { // slice if (target.validityBuffer != null) { @@ -651,6 +655,9 @@ public boolean isNull(int index) { * @return 1 if element at given index is not null, 0 otherwise */ public int isSet(int index) { + if (!hasNull) { + return 1; + } final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); final int bitIndex = index & 7; @@ -664,7 +671,10 @@ public int isSet(int index) { */ @Override public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); + if (hasNull) { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + return 0; } /** @@ -693,6 +703,15 @@ public void setNotNull(int index) { lastSet = index + 1; } + /** + * Set the flag to indicate if the vector has null. + * + * @param hasNull flag to indicate if the vector has null + */ + public void setHasNull(boolean hasNull) { + this.hasNull = hasNull; + } + /** * Start a new value in the list vector. *