From 6493ebf84cdcd0b6a059d3dd630ff130d292cf51 Mon Sep 17 00:00:00 2001 From: Siddharth Teotia Date: Sat, 14 Oct 2017 11:38:30 -0400 Subject: [PATCH 01/10] ARROW-1473: ValueVector new hierarchy prototype (implementation phase 1) Close #1164 Close #1198 Change-Id: If18e42d2edfdfef83e83621334a5b65a390e9db9 --- .../codegen/templates/ComplexReaders.java | 24 +- .../codegen/templates/ComplexWriters.java | 45 +- .../templates/NullableValueVectors.java | 542 ++++++++++-- .../main/codegen/templates/UnionVector.java | 18 +- .../vector/BaseNullableFixedWidthVector.java | 701 ++++++++++++++++ .../BaseNullableVariableWidthVector.java | 764 +++++++++++++++++ .../apache/arrow/vector/BaseValueVector.java | 20 + .../apache/arrow/vector/BitVectorHelper.java | 60 ++ .../arrow/vector/NullableIntVector.java | 299 +++++++ .../arrow/vector/NullableVarCharVector.java | 451 ++++++++++ .../org/apache/arrow/vector/ValueVector.java | 6 + .../apache/arrow/vector/VectorUnloader.java | 15 +- .../org/apache/arrow/vector/ZeroVector.java | 9 + .../vector/complex/FixedSizeListVector.java | 27 +- .../arrow/vector/complex/ListVector.java | 23 +- .../arrow/vector/complex/MapVector.java | 33 +- .../vector/file/json/JsonFileReader.java | 2 +- .../vector/TestBufferOwnershipTransfer.java | 6 +- .../arrow/vector/TestDictionaryVector.java | 59 +- .../arrow/vector/TestFixedSizeListVector.java | 13 +- .../arrow/vector/TestSplitAndTransfer.java | 15 +- .../apache/arrow/vector/TestValueVector.java | 788 +++++++++++++----- .../arrow/vector/TestVectorReAlloc.java | 11 +- .../arrow/vector/TestVectorUnloadLoad.java | 104 ++- .../arrow/vector/file/BaseFileTest.java | 70 +- .../arrow/vector/file/TestArrowFile.java | 2 +- 26 files changed, 3658 insertions(+), 449 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 38cd1bfdeb3..79106490515 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -70,7 +70,11 @@ public Field getField(){ public boolean isSet(){ <#if nullMode == "Nullable"> - return !vector.getAccessor().isNull(idx()); + <#if minor.class != "Int" && minor.class != "VarChar"> + return !vector.getAccessor().isNull(idx()); + <#else> + return !vector.isNull(idx()); + <#else> return true; @@ -93,11 +97,19 @@ public void read(${minor.class?cap_first}Holder h){ public void read(Nullable${minor.class?cap_first}Holder h){ - vector.getAccessor().get(idx(), h); + <#if minor.class != "Int" && minor.class != "VarChar"> + vector.getAccessor().get(idx(), h); + <#else> + vector.get(idx(), h); + } public ${friendlyType} read${safeType}(){ - return vector.getAccessor().getObject(idx()); + <#if minor.class == "Int" || minor.class == "VarChar"> + return vector.getObject(idx()); + <#else> + return vector.getAccessor().getObject(idx()); + } <#if minor.class == "TimeStampSec" || @@ -115,7 +127,11 @@ public void copyValue(FieldWriter w){ } public Object readObject(){ - return vector.getAccessor().getObject(idx()); + <#if minor.class == "Int" || minor.class == "VarChar"> + return (Object)vector.getObject(idx()); + <#else> + return vector.getAccessor().getObject(idx()); + } } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index fe099bede35..77f6594a078 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,11 +39,16 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - private final Nullable${name}Vector.Mutator mutator; + <#if minor.class != "Int" && minor.class != "VarChar"> + private final Nullable${name}Vector.Mutator mutator; + + final Nullable${name}Vector vector; public ${eName}WriterImpl(Nullable${name}Vector vector) { - this.mutator = vector.getMutator(); + <#if minor.class != "Int" && minor.class != "VarChar"> + this.mutator = vector.getMutator(); + this.vector = vector; } @@ -103,18 +108,33 @@ public void setPosition(int idx) { <#else> public void write(${minor.class}Holder h) { - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx(), h); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); + } public void write(Nullable${minor.class}Holder h) { - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx(), h); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); + } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setValueCount(idx()+1); + } <#if minor.class == "Decimal"> @@ -126,8 +146,13 @@ public void write(Nullable${minor.class}Holder h) { <#if mode == "Nullable"> public void writeNull() { - mutator.setNull(idx()); - vector.getMutator().setValueCount(idx()+1); + <#if minor.class != "Int" && minor.class != "VarChar"> + mutator.setNull(idx()); + vector.getMutator().setValueCount(idx()+1); + <#else> + vector.setNull(idx()); + vector.setValueCount(idx()+1); + } diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 122cd236482..5d1f5a33b90 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -19,8 +19,14 @@ <#list vv.types as type> <#list type.minor as minor> +<#if minor.class == "Int" || minor.class == "VarChar"> +<#assign className = "LegacyNullable${minor.class}Vector" /> +<#assign valuesName = "Nullable${minor.class}Vector" /> +<#else> <#assign className = "Nullable${minor.class}Vector" /> <#assign valuesName = "${minor.class}Vector" /> + + <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> @@ -44,15 +50,24 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, NullableVector, FieldVector { +<#if minor.class == "Int" || minor.class == "VarChar"> +@Deprecated + +public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); protected final static byte[] emptyByteArray = new byte[]{}; + + <#if minor.class != "Int" && minor.class != "VarChar"> private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this); + private final String bitsField = "$bits$"; private final String valuesField = "$values$"; + + <#if minor.class != "Int" && minor.class != "VarChar"> private final Field field; + final BitVector bits = new BitVector(bitsField, allocator); final ${valuesName} values; @@ -60,7 +75,9 @@ public final class ${className} extends BaseValueVector implements <#if type.maj private final Mutator mutator; private final Accessor accessor; + <#if minor.class != "Int" && minor.class != "VarChar"> private final List innerVectors; + <#if minor.typeParams??> <#assign typeParams = minor.typeParams?reverse> @@ -105,6 +122,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj this.mutator = new Mutator(); this.accessor = new Accessor(); + <#if minor.class != "Int" && minor.class != "VarChar"> this.field = new Field(name, fieldType, null); innerVectors = Collections.unmodifiableList(Arrays.asList( bits, @@ -113,16 +131,24 @@ public final class ${className} extends BaseValueVector implements <#if type.maj values )); + } - @Override + <#if minor.class != "Int" && minor.class != "VarChar"> + /* not needed for new vectors */ public BitVector getValidityVector() { return bits; } + @Override public List getFieldInnerVectors() { - return innerVectors; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getFieldInnerVectors(); + <#else> + return innerVectors; + } @Override @@ -139,6 +165,10 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.loadFieldBuffers(fieldNode, ownBuffers); + <#else> <#if type.major = "VarLen"> // variable width values: truncate offset vector buffer to size (#1) org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, @@ -151,34 +181,64 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); bits.valueCount = fieldNode.getLength(); + } public List getFieldBuffers() { - return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getFieldBuffers(); + <#else> + return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); + } @Override public Field getField() { - return field; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getField(); + <#else> + return field; + } @Override public MinorType getMinorType() { - return MinorType.${minor.class?upper_case}; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getMinorType(); + <#else> + return MinorType.${minor.class?upper_case}; + } @Override public FieldReader getReader(){ - return reader; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getReader(); + <#else> + return reader; + } @Override public int getValueCapacity(){ - return Math.min(bits.getValueCapacity(), values.getValueCapacity()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueCapacity(); + <#else> + return Math.min(bits.getValueCapacity(), values.getValueCapacity()); + } @Override public ArrowBuf[] getBuffers(boolean clear) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBuffers(clear); + <#else> final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class); if (clear) { for (final ArrowBuf buffer:buffers) { @@ -187,25 +247,41 @@ public ArrowBuf[] getBuffers(boolean clear) { clear(); } return buffers; + } @Override public void close() { - bits.close(); - values.close(); - super.close(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.close(); + <#else> + bits.close(); + values.close(); + super.close(); + } @Override public void clear() { - bits.clear(); - values.clear(); - super.clear(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.clear(); + <#else> + bits.clear(); + values.clear(); + super.clear(); + } @Override public int getBufferSize(){ - return values.getBufferSize() + bits.getBufferSize(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSize(); + <#else> + return values.getBufferSize() + bits.getBufferSize(); + } @Override @@ -214,34 +290,52 @@ public int getBufferSizeFor(final int valueCount) { return 0; } - return values.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSizeFor(valueCount); + <#else> + return values.getBufferSizeFor(valueCount) + + bits.getBufferSizeFor(valueCount); + } public ArrowBuf getBuffer() { return values.getDataBuffer(); } - @Override public ${valuesName} getValuesVector() { return values; } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); - values.setInitialCapacity(numRecords); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.setInitialCapacity(numRecords); + <#else> + bits.setInitialCapacity(numRecords); + values.setInitialCapacity(numRecords); + } @Override public void allocateNew() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(); + <#else> if(!allocateNewSafe()){ throw new OutOfMemoryException("Failure while allocating buffer."); } + } @Override public boolean allocateNewSafe() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.allocateNewSafe(); + <#else> /* Boolean to keep track if all the memory allocations were successful * Used in the case of composite vectors when we need to allocate multiple * buffers for multiple vectors. If one of the allocations failed we need to @@ -259,23 +353,38 @@ public boolean allocateNewSafe() { mutator.reset(); accessor.reset(); return success; + } @Override public void reAlloc() { - bits.reAlloc(); - values.reAlloc(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.reAlloc(); + <#else> + bits.reAlloc(); + values.reAlloc(); + } public void reset() { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.reset(); + <#else> bits.zeroVector(); mutator.reset(); accessor.reset(); + } <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(totalBytes, valueCount); + <#else> try { values.allocateNew(totalBytes, valueCount); bits.allocateNew(valueCount); @@ -286,6 +395,7 @@ public void allocateNew(int totalBytes, int valueCount) { bits.zeroVector(); mutator.reset(); accessor.reset(); + } @Override @@ -301,6 +411,10 @@ public int getCurrentSizeInBytes(){ <#else> @Override public void allocateNew(int valueCount) { + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(valueCount); + <#else> try { values.allocateNew(valueCount); bits.allocateNew(valueCount); @@ -311,6 +425,7 @@ public void allocateNew(int valueCount) { bits.zeroVector(); mutator.reset(); accessor.reset(); + } /** @@ -318,32 +433,86 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - bits.zeroVector(); - values.zeroVector(); + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.zeroVector(); + <#else> + bits.zeroVector(); + values.zeroVector(); + } + + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator, callBack); + <#else> return getTransferPair(ref, allocator); + } + + @Override public TransferPair getTransferPair(BufferAllocator allocator){ - return new TransferImpl(name, allocator); - + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(allocator); + <#else> + return new TransferImpl(name, allocator); + } + + @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator); + <#else> + return new TransferImpl(ref, allocator); + } + + @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${className}) to); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.makeTransferPair(to); + <#else> + return new TransferImpl((${className}) to); + + } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void transferTo(${valuesName} target) { + /* DELEGATE TO NEW VECTOR */ + <#if type.major == "VarLen"> + values.transferTo((BaseNullableVariableWidthVector) target); + <#else> + values.transferTo((BaseNullableFixedWidthVector) target); + + } + + public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) { + /* DELEGATE TO NEW VECTOR */ + <#if type.major == "VarLen"> + values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target); + <#else> + values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target); + } + <#else> public void transferTo(${className} target){ bits.transferTo(target.bits); values.transferTo(target.values); @@ -360,7 +529,11 @@ public void splitAndTransferTo(int startIndex, int length, ${className} target) target.mutator.lastSet = length - 1; } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> private class TransferImpl implements TransferPair { ${className} to; @@ -392,6 +565,9 @@ public void copyValueSafe(int fromIndex, int toIndex) { to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } + + + @Override public Accessor getAccessor(){ @@ -403,7 +579,20 @@ public Mutator getMutator(){ return mutator; } - public void copyFrom(int fromIndex, int thisIndex, ${className} from){ + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { + /* DELEGATE TO NEW VECTOR */ + values.copyFrom(fromIndex, thisIndex, from); + } + + public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { + /* DELEGATE TO NEW VECTOR */ + values.copyFromSafe(fromIndex, thisIndex, from); + } + <#else> + public void copyFrom(int fromIndex, int thisIndex, ${className} from) { final Accessor fromAccessor = from.getAccessor(); if (!fromAccessor.isNull(fromIndex)) { mutator.set(thisIndex, fromAccessor.get(fromIndex)); @@ -428,17 +617,28 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ values.copyFromSafe(fromIndex, thisIndex, from.values); <#if type.major == "VarLen">mutator.lastSet = thisIndex; } + @Override public long getValidityBufferAddress() { /* address of the databuffer associated with the bitVector */ - return (bits.getDataBuffer().memoryAddress()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBufferAddress(); + <#else> + return (bits.getDataBuffer().memoryAddress()); + } @Override public long getDataBufferAddress() { /* address of the dataBuffer associated with the valueVector */ - return (values.getDataBuffer().memoryAddress()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getDataBufferAddress(); + <#else> + return (bits.getDataBuffer().memoryAddress()); + } @Override @@ -446,17 +646,26 @@ public long getOffsetBufferAddress() { /* address of the dataBuffer associated with the offsetVector * this operation is not supported for fixed-width vector types. */ - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBufferAddress(); <#else> - return (values.getOffsetAddr()); + <#if type.major != "VarLen"> + throw new UnsupportedOperationException(); + <#else> + return (values.getOffsetAddr()); + } @Override public ArrowBuf getValidityBuffer() { - /* dataBuffer associated with the bitVector */ - return (bits.getDataBuffer()); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBuffer(); + <#else> + return (bits.getDataBuffer()); + } @Override @@ -468,10 +677,15 @@ public ArrowBuf getDataBuffer() { @Override public ArrowBuf getOffsetBuffer() { /* dataBuffer associated with the offsetVector of the valueVector */ - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBuffer(); <#else> - return (values.getOffsetBuffer()); + <#if type.major != "VarLen"> + throw new UnsupportedOperationException(); + <#else> + return (values.getOffsetBuffer()); + } @@ -485,38 +699,80 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m * @param index position of the value * @return value of the element, if not null */ - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - if (isNull(index)) { + <#if minor.class == "Int" || minor.class == "VarChar"> + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.get(index); + } + <#else> + + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + if (isNull(index)) { throw new IllegalStateException("Can't get a null value"); + } + return vAccessor.get(index); } - return vAccessor.get(index); - } + @Override public boolean isNull(int index) { - return isSet(index) == 0; + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.isNull(index); + <#else> + return isSet(index) == 0; + } public int isSet(int index){ - return bAccessor.get(index); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.isSet(index); + <#else> + return bAccessor.get(index); + } <#if type.major == "VarLen"> public long getStartEnd(int index){ - return vAccessor.getStartEnd(index); + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getStartEnd(index); + <#else> + return vAccessor.getStartEnd(index); + } @Override public int getValueLength(int index) { - return values.getAccessor().getValueLength(index); + <#if minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueLength(index); + <#else> + return values.getAccessor().getValueLength(index); + } + <#if minor.class == "Int" || minor.class == "VarChar"> + public void get(int index, Nullable${minor.class}Holder holder){ + /* DELEGATE TO NEW VECTOR */ + values.get(index, holder); + } + <#else> public void get(int index, Nullable${minor.class}Holder holder){ vAccessor.get(index, holder); holder.isSet = bAccessor.get(index); } + + <#if minor.class == "Int" || minor.class == "VarChar"> + @Override + public ${friendlyType} getObject(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.getObject(index); + } + <#else> @Override public ${friendlyType} getObject(int index) { if (isNull(index)) { @@ -525,6 +781,7 @@ public void get(int index, Nullable${minor.class}Holder holder){ return vAccessor.getObject(index); } } + <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> public StringBuilder getAsStringBuilder(int index) { @@ -538,7 +795,12 @@ public StringBuilder getAsStringBuilder(int index) { @Override public int getValueCount(){ - return bits.getAccessor().getValueCount(); + <#if minor.class == "Int" || minor.class == "VarChar"> + /* DELEGATE TO NEW VECTOR */ + return values.getValueCount(); + <#else> + return bits.getAccessor().getValueCount(); + } public void reset(){} @@ -551,21 +813,35 @@ public final class Mutator extends BaseDataValueVector.BaseMutator implements Nu private Mutator(){ } - public ${valuesName} getVectorWithValues(){ + public ${valuesName} getVectorWithValues() { return values; } + @Override public void setIndexDefined(int index){ + <#if minor.class == "Int" || minor.class == "VarChar"> + values.setIndexDefined(index); + <#else> bits.getMutator().setToOne(index); + } + + /** * Set the variable length element at the specified index to the supplied byte array. * * @param index position of the bit to set * @param value array of bytes (or int if smaller than 4 bytes) to write */ + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, value); + } + <#else> public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { setCount++; final ${valuesName}.Mutator valuesMutator = values.getMutator(); @@ -579,9 +855,24 @@ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.widt valuesMutator.set(index, value); <#if type.major == "VarLen">lastSet = index; } + + + <#if type.major == "VarLen"> + <#if minor.class == "VarChar"> + public void fillEmpties(int index) { + /* DELEGATE TO NEW VECTOR */ + values.fillEmpties(index); + } + @Override + public void setValueLengthSafe(int index, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setValueLengthSafe(index, length); + } + + <#else> public void fillEmpties(int index){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); for (int i = lastSet + 1; i < index; i++) { @@ -599,7 +890,16 @@ public void setValueLengthSafe(int index, int length) { lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setSafe(int index, byte[] value, int start, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value, start, length); + } + <#else> public void setSafe(int index, byte[] value, int start, int length) { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -612,7 +912,16 @@ public void setSafe(int index, byte[] value, int start, int length) { <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "VarChar"> + public void setSafe(int index, ByteBuffer value, int start, int length) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value, start, length); + } + <#else> public void setSafe(int index, ByteBuffer value, int start, int length) { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -625,11 +934,25 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setNull(int index) { + /* DELEGATE TO NEW VECTOR */ + values.setNull(index); + } + <#else> public void setNull(int index){ bits.getMutator().setSafe(index, 0); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* these methods are probably not needed */ public void setSkipNull(int index, ${minor.class}Holder holder){ values.getMutator().set(index, holder); } @@ -637,8 +960,17 @@ public void setSkipNull(int index, ${minor.class}Holder holder){ public void setSkipNull(int index, Nullable${minor.class}Holder holder){ values.getMutator().set(index, holder); } + + - public void set(int index, Nullable${minor.class}Holder holder){ + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, Nullable${minor.class}Holder holder) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, holder); + } + <#else> + public void set(int index, Nullable${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -649,8 +981,17 @@ public void set(int index, Nullable${minor.class}Holder holder){ valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index; } + + + - public void set(int index, ${minor.class}Holder holder){ + <#if minor.class == "Int" || minor.class == "VarChar"> + public void set(int index, ${minor.class}Holder holder) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, holder); + } + <#else> + public void set(int index, ${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -661,11 +1002,45 @@ public void set(int index, ${minor.class}Holder holder){ valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index; } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public boolean isSafe(int outIndex) { + /* DELEGATE TO NEW VECTOR */ + return values.isSafe(outIndex); + } + <#else> public boolean isSafe(int outIndex) { return outIndex < ${className}.this.getValueCapacity(); } + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + <#if minor.class == "Int"> + public void set(int index, int isSet, int valueField) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, isSet, valueField); + } + public void setSafe(int index, int isSet, int valueField) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, isSet, valueField); + } + + <#if minor.class == "VarChar"> + public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /* DELEGATE TO NEW VECTOR */ + values.set(index, isSet, startField, endField, bufferField); + } + public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, isSet, startField, endField, bufferField); + } + + <#else> <#assign fields = minor.fields!type.fields /> public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ final ${valuesName}.Mutator valuesMutator = values.getMutator(); @@ -688,8 +1063,21 @@ public void setSafe(int index, int isSet<#list fields as field><#if field.includ setCount++; <#if type.major == "VarLen">lastSet = index; } + + + <#if minor.class == "Int" || minor.class == "VarChar"> + public void setSafe(int index, Nullable${minor.class}Holder value) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + } + + public void setSafe(int index, ${minor.class}Holder value) { + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + } + <#else> public void setSafe(int index, Nullable${minor.class}Holder value) { <#if type.major == "VarLen"> fillEmpties(index); @@ -709,15 +1097,25 @@ public void setSafe(int index, ${minor.class}Holder value) { setCount++; <#if type.major == "VarLen">lastSet = index; } + + + <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { + <#if minor.class == "Int"> + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); + <#else> bits.getMutator().setSafeToOne(index); values.getMutator().setSafe(index, value); setCount++; + } - + + + <#if minor.class == "Decimal"> public void set(int index, ${friendlyType} value) { bits.getMutator().setToOne(index); @@ -729,8 +1127,17 @@ public void setSafe(int index, ${friendlyType} value) { values.getMutator().setSafe(index, value); setCount++; } - + + + + <#if minor.class == "Int" || minor.class == "VarChar"> + @Override + public void setValueCount(int valueCount) { + /* DELEGATE TO NEW VECTOR */ + values.setValueCount(valueCount); + } + <#else> @Override public void setValueCount(int valueCount) { assert valueCount >= 0; @@ -740,7 +1147,12 @@ public void setValueCount(int valueCount) { values.getMutator().setValueCount(valueCount); bits.getMutator().setValueCount(valueCount); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ @Override public void generateTestData(int valueCount){ bits.getMutator().generateTestDataAlt(valueCount); @@ -748,13 +1160,27 @@ public void generateTestData(int valueCount){ <#if type.major = "VarLen">lastSet = valueCount; setValueCount(valueCount); } + + + + <#if minor.class != "Int" && minor.class != "VarChar"> + /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ @Override public void reset(){ setCount = 0; <#if type.major = "VarLen">lastSet = -1; } + + + + <#if minor.class == "VarChar"> + public void setLastSet(int value) { + /* DELEGATE TO NEW VECTOR */ + values.setLastSet(value); + } + <#else> public void setLastSet(int value) { <#if type.major = "VarLen"> lastSet = value; @@ -762,7 +1188,16 @@ public void setLastSet(int value) { throw new UnsupportedOperationException(); } + + + + <#if minor.class == "VarChar"> + public int getLastSet() { + /* DELEGATE TO NEW VECTOR */ + return values.getLastSet(); + } + <#else> public int getLastSet() { <#if type.major != "VarLen"> throw new UnsupportedOperationException(); @@ -770,6 +1205,7 @@ public int getLastSet() { return lastSet; } + } } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index fe24a8674bd..3c7ed019019 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -436,7 +436,11 @@ public Object getObject(int index) { <#assign uncappedName = name?uncap_first/> <#if !minor.typeParams?? > case ${name?upper_case}: - return get${name}Vector().getAccessor().getObject(index); + <#if minor.class != "Int" && minor.class != "VarChar"> + return get${name}Vector().getAccessor().getObject(index); + <#else> + return get${name}Vector().getObject(index); + @@ -530,7 +534,11 @@ public void setSafe(int index, UnionHolder holder) { <#if !minor.typeParams?? > public void setSafe(int index, Nullable${name}Holder holder) { setType(index, MinorType.${name?upper_case}); - get${name}Vector().getMutator().setSafe(index, holder); + <#if minor.class != "Int" && minor.class != "VarChar"> + get${name}Vector().getMutator().setSafe(index, holder); + <#else> + get${name}Vector().setSafe(index, holder); + } @@ -547,4 +555,10 @@ public void reset() { } @Override public void generateTestData(int values) { } } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java new file mode 100644 index 00000000000..c5f781000c9 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -0,0 +1,701 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +public abstract class BaseNullableFixedWidthVector extends BaseValueVector + implements FixedWidthVector, FieldVector { + private final byte typeWidth; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + + protected final Field field; + private int allocationMonitor; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected int valueCount; + + public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType, final byte typeWidth) { + super(name, allocator); + this.typeWidth = typeWidth; + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + field = new Field(name, fieldType, null); + valueCount = 0; + allocationMonitor = 0; + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + } + + /* TODO: + * Determine how writerIndex and readerIndex need to be used. Right now we + * are setting the writerIndex and readerIndex in the call to getFieldBuffers + * using the valueCount -- this assumes that the caller of getFieldBuffers + * on the vector has already invoked setValueCount. + * + * Do we need to set them during vector transfer and splitAndTransfer? + */ + + /* TODO: + * + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy and those vectors have not + * yet been refactored so moving things to the top class as of now + * is not a good idea. + */ + + /* TODO: + * See if we need logger -- unnecessary object probably + */ + + protected abstract org.slf4j.Logger getLogger(); + + @Override + public Mutator getMutator() { + throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + } + + @Override + public Accessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + } + + @Override + public long getValidityBufferAddress() { + return (validityBuffer.memoryAddress()); + } + + @Override + public long getDataBufferAddress() { + return (valueBuffer.memoryAddress()); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long)valueCount * typeWidth; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int)size; + validityAllocationSizeInBytes = getSizeFromCount(valueCount); + } + + @Override + public int getValueCapacity(){ + return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); + } + + /* for test purposes */ + private int getValueBufferValueCapacity() { + return (int)((valueBuffer.capacity() * 1.0)/typeWidth); + } + + /* for test purposes */ + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + /* number of bytes for the validity buffer for the given valueCount */ + protected int getSizeFromCount(int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } + + @Override + public void zeroVector() { + initValidityBuffer(); + initValueBuffer(); + } + + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + private void initValueBuffer() { + valueBuffer.setZero(0, valueBuffer.capacity()); + } + + public void reset() { + zeroVector(); + } + + @Override + public void close() { clear(); } + + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + } + + /* used to step down the memory allocation */ + protected void incrementAllocationMonitor() { + if (allocationMonitor < 0) { + allocationMonitor = 0; + } + allocationMonitor++; + } + + /* used to step up the memory allocation */ + protected void decrementAllocationMonitor() { + if (allocationMonitor > 0) { + allocationMonitor = 0; + } + allocationMonitor--; + } + + @Override + public void allocateNew() { + if(!allocateNewSafe()){ + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try{ + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + return false; + } + + return true; + } + + public void allocateNew(int valueCount) { + long valueBufferSize = valueCount * typeWidth; + long validityBufferSize = getSizeFromCount(valueCount); + + if (allocationMonitor > 10) { + /* step down the default memory allocation since we have observed + * multiple times that provisioned value capacity was much larger than + * actually needed. see setValueCount for more details. + */ + valueBufferSize = Math.max(8, valueBufferSize / 2); + validityBufferSize = Math.max(8, validityBufferSize / 2); + allocationMonitor = 0; + } else if (allocationMonitor < -2) { + valueBufferSize = valueBufferSize * 2L; + validityBufferSize = validityBufferSize * 2L; + allocationMonitor = 0; + } + + if (valueBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(valueBufferSize, validityBufferSize); + } catch(Exception e) { + getLogger().error("ERROR: Failure in allocateNew"); + getLogger().error(e.getMessage()); + clear(); + throw e; + } + } + + /** + * Actual memory allocation is done by this function. All the calculations + * and knowledge about what size to allocate is upto the callers of this + * method. + * Callers appropriately handle errors if memory allocation fails here. + * Callers should also take care of determining that desired size is + * within the bounds of max allocation allowed and any other error + * conditions. + */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { + /* allocate data buffer */ + int curSize = (int)valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + + /* allocate validity buffer */ + allocateValidityBuffer((int)validityBufferSize); + initValidityBuffer(); + } + + /* + * during splitAndTransfer, if we splitting from a random position within a byte, + * we can't just slice the source buffer so we have to explicitly allocate the + * validityBuffer of the target vector. This is unlike the databuffer which we can + * always slice for the target vector. + */ + private void allocateValidityBuffer(final int validityBufferSize) { + validityBuffer = allocator.buffer(validityBufferSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = validityBufferSize; + initValidityBuffer(); + } + + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { return 0; } + return (count * typeWidth) + getSizeFromCount(count); + } + + @Override + public int getBufferSize() { + if (valueCount == 0) { return 0; } + return (valueCount * typeWidth) + getSizeFromCount(valueCount); + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = new ArrowBuf[2]; + buffers[0] = validityBuffer; + buffers[1] = valueBuffer; + if (clear) { + for (final ArrowBuf buffer:buffers) { + buffer.retain(1); + } + clear(); + } + return buffers; + } + + @Override + public void reAlloc() { + valueBuffer = reallocBufferHelper(valueBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (dataBuffer ? valueAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", + name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes), + newAllocationSize); + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (dataBuffer) { + valueAllocationSizeInBytes = (int)newAllocationSize; + } + else { + validityAllocationSizeInBytes = (int)newAllocationSize; + } + + return buffer; + } + + @Override + public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf dataBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + valueCount = fieldNode.getLength(); + + valueAllocationSizeInBytes = valueBuffer.capacity(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + public List getFieldBuffers() { + List result = new ArrayList<>(2); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getSizeFromCount(valueCount)); + valueBuffer.readerIndex(0); + valueBuffer.writerIndex(valueCount * typeWidth); + + result.add(validityBuffer); + result.add(valueBuffer); + + return result; + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator){ + return getTransferPair(name, allocator); + } + + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + public void transferTo(BaseNullableFixedWidthVector target){ + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + clear(); + } + + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferValueBuffer(startIndex, length, target); + target.setValueCount(length); + } + + private void splitAndTransferValueBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + final int startPoint = startIndex * typeWidth; + final int sliceLength = length * typeWidth; + target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; + } + + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + int count = 0; + final int sizeInBytes = getSizeFromCount(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + + /** + * Get the value count of vector. This will always be zero unless + * {@link #setValueCount(int)} has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount(){ + return valueCount; + } + + + /** + * Set value count for the vector. + * + * @param valueCount value count to set + */ + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + final int currentValueCapacity = getValueCapacity(); + while (valueCount > getValueCapacity()) { + reAlloc(); + } + /* + * We are trying to understand the pattern of memory allocation. + * If initially, the user did vector.allocateNew(), we would have + * allocated memory of default size (4096 * type width). + * Later on user invokes setValueCount(count). + * + * If the existing value capacity is twice as large as the + * valueCount, we know that we over-provisioned memory in the + * first place when default memory allocation was done because user + * really needs a much less value count in the vector. + * + * We record this by bumping up the allocationMonitor. If this pattern + * happens for certain number of times and allocationMonitor + * reaches the threshold (internal hardcoded) value, subsequent + * call to allocateNew() will take care of stepping down the + * default memory allocation size. + * + * Another case would be under-provisioning the initial memory and + * thus going through a lot of realloc(). Here the goal is to + * see if we can minimize the number of reallocations. Again the + * state is recorded in allocationMonitor by decrementing it + * (negative value). If a threshold is hit, realloc will try to + * allocate more memory in order to possibly avoid a future realloc. + * This case is also applicable to setSafe() methods which can trigger + * a realloc() and thus we record the state there as well. + */ + if (valueCount > 0) { + if (currentValueCapacity >= (valueCount * 2)) { + incrementAllocationMonitor(); + } else if (currentValueCapacity <= (valueCount/2)) { + decrementAllocationMonitor(); + } + } + } + + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + public void setIndexDefined(int index) { + handleSafe(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void set(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void set(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + + protected void handleSafe(int index) { + while (index >= getValueCapacity()) { + decrementAllocationMonitor(); + reAlloc(); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java new file mode 100644 index 00000000000..a79709da4dd --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -0,0 +1,764 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public abstract class BaseNullableVariableWidthVector extends BaseValueVector + implements VariableWidthVector, FieldVector { + private static final int DEFAULT_RECORD_BYTE_COUNT = 8; + private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + private int offsetAllocationSizeInBytes; + + /* protected members */ + protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + protected static final byte[] emptyByteArray = new byte[]{}; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected ArrowBuf offsetBuffer; + protected int valueCount; + protected int lastSet; + protected final Field field; + private boolean cleared; + + public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType) { + super(name, allocator); + valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; + validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; + field = new Field(name, fieldType, null); + valueCount = 0; + lastSet = -1; + offsetBuffer = allocator.getEmpty(); + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + cleared = false; + } + + /* TODO: + * Determine how writerIndex and readerIndex need to be used. Right now we + * are setting the writerIndex and readerIndex in the call to getFieldBuffers + * using the valueCount -- this assumes that the caller of getFieldBuffers + * on the vector has already invoked setValueCount. + * + * Do we need to set them during vector transfer and splitAndTransfer? + */ + + /* TODO: + * + * see if getNullCount() can be made faster -- O(1) + */ + + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy and those vectors have not + * yet been refactored so moving things to the top class as of now + * is not a good idea. + */ + + /* TODO: + * See if we need logger -- unnecessary object probably + */ + + /* TODO: + * Implement getBufferSize(), getCurrentSizeInBytes(). + */ + + protected abstract org.slf4j.Logger getLogger(); + + public VariableWidthMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + } + + public VariableWidthAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + @Override + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } + + @Override + public long getOffsetBufferAddress() { + return offsetBuffer.memoryAddress(); + } + + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + @Override + public long getDataBufferAddress() { + return valueBuffer.memoryAddress(); + } + + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int)size; + validityAllocationSizeInBytes = getSizeFromCount(valueCount); + /* to track the end offset of last data element in vector, we need + * an additional slot in offset buffer. + */ + offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; + } + + @Override + public int getValueCapacity(){ + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + /* for test purposes */ + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + /* for test purposes */ + private int getOffsetBufferValueCapacity() { + return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + } + + /* number of bytes for the validity buffer for a given valueCount */ + protected int getSizeFromCount(int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } + + public void zeroVector() { + initValidityBuffer(); + initOffsetBuffer(); + } + + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + private void initOffsetBuffer() { + offsetBuffer.setZero(0, offsetBuffer.capacity()); + } + + public void reset() { + zeroVector(); + lastSet = -1; + } + + @Override + public void close() { + clear(); + } + + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + offsetBuffer = releaseBuffer(offsetBuffer); + cleared = true; + lastSet = -1; + valueCount = 0; + } + + @Override + public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + ArrowBuf dataBuffer = ownBuffers.get(2); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + lastSet = fieldNode.getLength() - 1; + valueCount = fieldNode.getLength(); + } + + public List getFieldBuffers() { + List result = new ArrayList<>(3); + final int lastDataOffset = getstartOffset(valueCount); + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getSizeFromCount(valueCount)); + offsetBuffer.readerIndex(0); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + valueBuffer.readerIndex(0); + valueBuffer.writerIndex(lastDataOffset); + + result.add(validityBuffer); + result.add(offsetBuffer); + result.add(valueBuffer); + + return result; + } + + @Override + public void allocateNew() { + if(!allocateNewSafe()){ + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + long curAllocationSizeOffset = offsetAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || + curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + return false; + } + + return true; + } + + @Override + public void allocateNew(int totalBytes, int valueCount) { + assert totalBytes >= 0; + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getSizeFromCount(valueCount); + + if (totalBytes > MAX_ALLOCATION_SIZE || + offsetBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } + + /* we are doing a new allocation -- release the current buffers */ + clear(); + + try { + allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); + } catch (Exception e) { + getLogger().error("ERROR: Failure in allocateNewSafe"); + getLogger().error(e.getMessage()); + clear(); + } + } + + private void allocateBytes(final long valueBufferSize, final long validityBufferSize, + final long offsetBufferSize) { + /* allocate data buffer */ + int curSize = (int)valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + allocateValidityBuffer(validityBufferSize); + allocateOffsetBuffer(offsetBufferSize); + } + + private void allocateOffsetBuffer(final long size) { + final int curSize = (int)size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + initOffsetBuffer(); + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + initValidityBuffer(); + } + + public void reAlloc() { + reallocValueBuffer(); + reallocValidityAndOffsetBuffers(); + } + + protected void reallocValueBuffer() { + long baseSize = valueAllocationSizeInBytes; + final int currentBufferCapacity = valueBuffer.capacity(); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); + valueBuffer.release(); + valueBuffer = newBuf; + valueAllocationSizeInBytes = (int)newAllocationSize; + } + + protected void reallocValidityAndOffsetBuffers() { + offsetBuffer = reallocBufferHelper(offsetBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /* need to refactor this to keep the logic in an single place and make callers + * more intelligent. see handleSafe() for more comments on realloc + */ + + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", + name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes), + newAllocationSize); + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (offsetBuffer) { + offsetAllocationSizeInBytes = (int)newAllocationSize; + } + else { + validityAllocationSizeInBytes = (int)newAllocationSize; + } + + return buffer; + } + + @Override + public int getByteCapacity(){ + return valueBuffer.capacity(); + } + + @Override + public int getCurrentSizeInBytes(){ + /* TODO */ + return 0; + } + + @Override + public int getBufferSize() { + /* TODO */ + return 0; + } + + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + + final int validityBufferSize = getSizeFromCount(valueCount); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + /* get the end offset for this valueCount */ + final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + return validityBufferSize + offsetBufferSize + dataBufferSize; + } + + @Override + public Field getField() { + return field; + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = new ArrowBuf[3]; + buffers[0] = validityBuffer; + buffers[1] = offsetBuffer; + buffers[1] = valueBuffer; + if (clear) { + for (final ArrowBuf buffer:buffers) { + buffer.retain(1); + } + clear(); + } + return buffers; + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator){ + return getTransferPair(name, allocator); + } + + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + public void transferTo(BaseNullableVariableWidthVector target){ + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + target.setLastSet(lastSet); + clear(); + } + + public void splitAndTransferTo(int startIndex, int length, + BaseNullableVariableWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferOffsetBuffer(startIndex, length, target); + target.setLastSet(length - 1); + target.setValueCount(length); + } + + /* + * transfer the offsets along with data + */ + private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { + final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); + final int dataLength = end - start; + target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + for (int i = 0; i < length + 1; i++) { + final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; + target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); + } + target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableVariableWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + int count = 0; + final int sizeInBytes = getSizeFromCount(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the value count of vector. This will always be zero unless + * setValueCount(int) has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount(){ + return valueCount; + } + + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ + public void setValueCount(int valueCount) { + assert valueCount >= 0; + this.valueCount = valueCount; + while (valueCount > getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + fillHoles(valueCount); + lastSet = valueCount - 1; + } + + public void fillEmpties(int index) { + handleSafe(index, emptyByteArray.length); + fillHoles(index); + lastSet = index - 1; + } + + public void setLastSet(int value) { + lastSet = value; + } + + public int getLastSet() { + return lastSet; + } + + public long getStartEnd(int index) { + return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + public void setIndexDefined(int index) { + handleSafe(index, 0); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected final void fillHoles(int index) { + for (int i = lastSet + 1; i < index; i++) { + setBytes(i, emptyByteArray, 0, emptyByteArray.length); + } + lastSet = index - 1; + } + + protected final void setBytes(int index, byte[] value, int start, int length) { + /* end offset of current last element in the vector. this will + * be the start offset of new element we are trying to store. + */ + final int startOffset = getstartOffset(index); + /* set new end offset */ + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + /* store the var length data in value buffer */ + valueBuffer.setBytes(startOffset, value, start, length); + } + + protected final int getstartOffset(int index) { + return offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + protected final void handleSafe(int index, int dataLength) { + /* + * IMPORTANT: + * value buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * Here there is no concept of getValueCapacity() in the + * data stream. getValueCapacity() is applicable only to validity + * and offset buffers. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + */ + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + final int startOffset = getstartOffset(index); + while (valueBuffer.capacity() < (startOffset + dataLength)) { + reallocValueBuffer(); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 598e578e55a..1db96246485 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -18,8 +18,10 @@ package org.apache.arrow.vector; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import java.util.List; import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.memory.BufferAllocator; @@ -122,5 +124,23 @@ public static boolean checkBufRefs(final ValueVector vv) { public BufferAllocator getAllocator() { return allocator; } + + protected void compareTypes(BaseValueVector target, String caller) { + if (this.getMinorType() != target.getMinorType()) { + throw new UnsupportedOperationException(caller + " should have vectors of exact same type"); + } + } + + protected ArrowBuf releaseBuffer(ArrowBuf buffer) { + buffer.release(); + buffer = allocator.getEmpty(); + return buffer; + } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java new file mode 100644 index 00000000000..2439bd26eab --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; + +class BitVectorHelper { + + /** + * Get the index of byte corresponding to bit index in validity buffer + */ + protected static int byteIndex(int absoluteBitIndex) { + return absoluteBitIndex >> 3; + } + + /** + * Get the relative index of bit within the byte in validity buffer + */ + private static int bitIndex(int absoluteBitIndex) { + return absoluteBitIndex & 7; + } + + protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + currentByte |= bitMask; + validityBuffer.setByte(byteIndex, currentByte); + } + + protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + if (value != 0) { + currentByte |= bitMask; + } else { + currentByte -= (bitMask & currentByte); + } + validityBuffer.setByte(byteIndex, currentByte); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java new file mode 100644 index 00000000000..26b19fa7bd5 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -0,0 +1,299 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntHolder; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableIntVector implements a fixed width vector of values which could + * be null. A validity buffer (bit vector) is maintained to track which + * elements in the vector are null. + */ +public class NullableIntVector extends BaseNullableFixedWidthVector { + private static final org.slf4j.Logger logger = + org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), + allocator); + } + + public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntReaderImpl(NullableIntVector.this); + } + + @Override + protected org.slf4j.Logger getLogger() { + return logger; + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int valueField ) { + if (isSet > 0) { + set(index, valueField); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, int valueField ) { + handleSafe(index); + set(index, isSet, valueField); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntVector to){ + this.to = to; + } + + @Override + public NullableIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java new file mode 100644 index 00000000000..b9021544b0e --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -0,0 +1,451 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.VarCharReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.VarCharHolder; +import org.apache.arrow.vector.holders.NullableVarCharHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.TransferPair; + +import java.nio.ByteBuffer; + +public class NullableVarCharVector extends BaseNullableVariableWidthVector { + private static final org.slf4j.Logger logger = + org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); + private final FieldReader reader; + + public NullableVarCharVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); + } + + public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarCharReaderImpl(NullableVarCharVector.this); + } + + @Override + protected org.slf4j.Logger getLogger() { + return logger; + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARCHAR; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return Text object for non-null element, null otherwise + */ + public Text getObject(int index) { + Text result = new Text(); + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + result.set(b); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarCharHolder holder){ + assert index >= 0; + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + holder.isSet = 1; + holder.start = startOffset; + holder.end = dataLength; + holder.buffer = valueBuffer; + } + + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + + public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + fillHoles(thisIndex); + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { + fillEmpties(thisIndex); + if (from.isSet(fromIndex) != 0) { + setSafe(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index, 0); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField); + final ArrowBuf bb = bufferField.slice(startField, endField); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + assert index >= 0; + handleSafe(index, endField); + set(index, isSet, startField, endField, bufferField); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarCharVector)to); + } + + private class TransferImpl implements TransferPair { + NullableVarCharVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableVarCharVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarCharVector to){ + this.to = to; + } + + @Override + public NullableVarCharVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index fb7286f852c..e6048b4b517 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -255,4 +255,10 @@ interface Mutator { * @return buffer */ public ArrowBuf getOffsetBuffer(); + + public int getValueCount(); + + public void setValueCount(int valueCount); + + public Object getObject(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index f8385a7262a..96243eb5876 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -27,6 +27,8 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.schema.ArrowVectorType; +import javax.annotation.Nullable; + public class VectorUnloader { private final VectorSchemaRoot root; @@ -53,8 +55,17 @@ public ArrowRecordBatch getRecordBatch() { } private void appendNodes(FieldVector vector, List nodes, List buffers) { - Accessor accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); + Accessor accessor = null; + if (vector instanceof NullableIntVector) { + nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(), + includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1)); + } else if (vector instanceof NullableVarCharVector) { + nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(), + includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1)); + } else { + accessor = vector.getAccessor(); + nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); + } List fieldBuffers = vector.getFieldBuffers(); List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 5ac00375f83..fcef02f4a1b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -259,4 +259,13 @@ public ArrowBuf getDataBuffer() { public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } + + @Override + public int getValueCount() { return 0; } + + @Override + public void setValueCount(int valueCount) { } + + @Override + public Object getObject(int index) { return null; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index b3be37541c1..e95442acf6d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -34,14 +34,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; @@ -331,9 +324,15 @@ public Object getObject(int index) { return null; } final List vals = new JsonStringArrayList<>(listSize); - final ValueVector.Accessor valuesAccessor = vector.getAccessor(); - for (int i = 0; i < listSize; i++) { - vals.add(valuesAccessor.getObject(index * listSize + i)); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + for (int i = 0; i < listSize; i++) { + vals.add(vector.getObject(index * listSize + i)); + } + } else { + final ValueVector.Accessor valuesAccessor = vector.getAccessor(); + for (int i = 0; i < listSize; i++) { + vals.add(valuesAccessor.getObject(index * listSize + i)); + } } return vals; } @@ -367,7 +366,11 @@ public void setNotNull(int index) { @Override public void setValueCount(int valueCount) { bits.getMutator().setValueCount(valueCount); - vector.getMutator().setValueCount(valueCount * listSize); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + vector.setValueCount(valueCount * listSize); + } else { + vector.getMutator().setValueCount(valueCount * listSize); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index ea28a606192..4b2c9139219 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -40,6 +40,8 @@ import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.NullableIntVector; +import org.apache.arrow.vector.NullableVarCharVector; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -376,10 +378,18 @@ public Object getObject(int index) { final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); final int start = offsetsAccessor.get(index); final int end = offsetsAccessor.get(index + 1); - final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor(); - for (int i = start; i < end; i++) { - vals.add(valuesAccessor.getObject(i)); + final ValueVector vv = getDataVector(); + if (vv instanceof NullableIntVector || vv instanceof NullableVarCharVector) { + for (int i = start; i < end; i++) { + vals.add(vv.getObject(i)); + } + } else { + final ValueVector.Accessor valuesAccessor = vv.getAccessor(); + for (int i = start; i < end; i++) { + vals.add(valuesAccessor.getObject(i)); + } } + return vals; } @@ -432,7 +442,12 @@ public void setValueCount(int valueCount) { offsets.getMutator().setValueCount(valueCount + 1); } final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); + if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { + vector.setValueCount(childValueCount); + } else { + vector.getMutator().setValueCount(childValueCount); + } + bits.getMutator().setValueCount(valueCount); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index f46635ad760..95efa608718 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -35,9 +35,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; @@ -272,10 +270,19 @@ public Object getObject(int index) { Map vv = new JsonStringHashMap<>(); for (String child : getChildFieldNames()) { ValueVector v = getChild(child); - if (v != null && index < v.getAccessor().getValueCount()) { - Object value = v.getAccessor().getObject(index); - if (value != null) { - vv.put(child, value); + if (v instanceof NullableVarCharVector || v instanceof NullableIntVector) { + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); + } + } + } else { + if (v != null && index < v.getAccessor().getValueCount()) { + Object value = v.getAccessor().getObject(index); + if (value != null) { + vv.put(child, value); + } } } } @@ -302,7 +309,11 @@ public class Mutator extends BaseValueVector.BaseMutator { @Override public void setValueCount(int valueCount) { for (final ValueVector v : getChildren()) { - v.getMutator().setValueCount(valueCount); + if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) { + v.setValueCount(valueCount); + } else { + v.getMutator().setValueCount(valueCount); + } } MapVector.this.valueCount = valueCount; } @@ -362,4 +373,10 @@ public List getChildrenFromFields() { return getChildren(); } + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } + } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index e1c7c909f10..0de804406f7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -265,7 +265,7 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1); break; case VARCHAR: - ((NullableVarCharVector) vector).getMutator().setLastSet(count - 1); + ((NullableVarCharVector) vector).setLastSet(count - 1); break; } vector.getMutator().setValueCount(count); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 1a801a63ec6..4fe2861450a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -40,7 +40,7 @@ public void testTransferFixedWidth() { NullableIntVector v1 = new NullableIntVector("v1", childAllocator1); v1.allocateNew(); - v1.getMutator().setValueCount(4095); + v1.setValueCount(4095); NullableIntVector v2 = new NullableIntVector("v2", childAllocator2); @@ -60,8 +60,8 @@ public void testTransferVariableidth() { NullableVarCharVector v1 = new NullableVarCharVector("v1", childAllocator1); v1.allocateNew(); - v1.getMutator().setSafe(4094, "hello world".getBytes(), 0, 11); - v1.getMutator().setValueCount(4001); + v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setValueCount(4001); NullableVarCharVector v2 = new NullableVarCharVector("v2", childAllocator2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index f8c16e7fc81..1185246f88c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -56,24 +56,22 @@ public void testEncodeStrings() { // Create a new value vector try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(512, 5); // set some values - m.setSafe(0, zero, 0, zero.length); - m.setSafe(1, one, 0, one.length); - m.setSafe(2, one, 0, one.length); - m.setSafe(3, two, 0, two.length); - m.setSafe(4, zero, 0, zero.length); - m.setValueCount(5); + vector.setSafe(0, zero, 0, zero.length); + vector.setSafe(1, one, 0, one.length); + vector.setSafe(2, one, 0, one.length); + vector.setSafe(3, two, 0, two.length); + vector.setSafe(4, zero, 0, zero.length); + vector.setValueCount(5); // set some dictionary values - final NullableVarCharVector.Mutator m2 = dictionaryVector.getMutator(); dictionaryVector.allocateNew(512, 3); - m2.setSafe(0, zero, 0, zero.length); - m2.setSafe(1, one, 0, one.length); - m2.setSafe(2, two, 0, two.length); - m2.setValueCount(3); + dictionaryVector.setSafe(0, zero, 0, zero.length); + dictionaryVector.setSafe(1, one, 0, one.length); + dictionaryVector.setSafe(2, two, 0, two.length); + dictionaryVector.setValueCount(3); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); @@ -81,20 +79,20 @@ public void testEncodeStrings() { // verify indices assertEquals(NullableIntVector.class, encoded.getClass()); - NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor(); - assertEquals(5, indexAccessor.getValueCount()); - assertEquals(0, indexAccessor.get(0)); - assertEquals(1, indexAccessor.get(1)); - assertEquals(1, indexAccessor.get(2)); - assertEquals(2, indexAccessor.get(3)); - assertEquals(0, indexAccessor.get(4)); + NullableIntVector index = ((NullableIntVector)encoded); + assertEquals(5, index.getValueCount()); + assertEquals(0, index.get(0)); + assertEquals(1, index.get(1)); + assertEquals(1, index.get(2)); + assertEquals(2, index.get(3)); + assertEquals(0, index.get(4)); // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount()); + assertEquals(vector.getValueCount(), ((NullableVarCharVector)decoded).getValueCount()); for (int i = 0; i < 5; i++) { - assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i)); + assertEquals(vector.getObject(i), ((NullableVarCharVector)decoded).getObject(i)); } } } @@ -106,21 +104,20 @@ public void testEncodeLargeVector() { // Create a new value vector try (final NullableVarCharVector vector = newNullableVarCharVector("foo", allocator); final NullableVarCharVector dictionaryVector = newNullableVarCharVector("dict", allocator);) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(); int count = 10000; for (int i = 0; i < 10000; ++i) { - vector.getMutator().setSafe(i, data[i % 3], 0, data[i % 3].length); + vector.setSafe(i, data[i % 3], 0, data[i % 3].length); } - vector.getMutator().setValueCount(count); + vector.setValueCount(count); dictionaryVector.allocateNew(512, 3); - dictionaryVector.getMutator().setSafe(0, zero, 0, zero.length); - dictionaryVector.getMutator().setSafe(1, one, 0, one.length); - dictionaryVector.getMutator().setSafe(2, two, 0, two.length); - dictionaryVector.getMutator().setValueCount(3); + dictionaryVector.setSafe(0, zero, 0, zero.length); + dictionaryVector.setSafe(1, one, 0, one.length); + dictionaryVector.setSafe(2, two, 0, two.length); + dictionaryVector.setValueCount(3); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); @@ -129,10 +126,10 @@ public void testEncodeLargeVector() { // verify indices assertEquals(NullableIntVector.class, encoded.getClass()); - NullableIntVector.Accessor indexAccessor = ((NullableIntVector) encoded).getAccessor(); - assertEquals(count, indexAccessor.getValueCount()); + NullableIntVector index = ((NullableIntVector) encoded); + assertEquals(count, index.getValueCount()); for (int i = 0; i < count; ++i) { - assertEquals(i % 3, indexAccessor.get(i)); + assertEquals(i % 3, index.get(i)); } // now run through the decoder and verify we get the original back diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 43d9387b106..168deacd6a6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -54,15 +54,14 @@ public void terminate() throws Exception { public void testIntType() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); - NullableIntVector.Mutator mutator = nested.getMutator(); vector.allocateNew(); for (int i = 0; i < 10; i++) { vector.getMutator().setNotNull(i); - mutator.set(i * 2, i); - mutator.set(i * 2 + 1, i + 10); + nested.set(i * 2, i); + nested.set(i * 2 + 1, i + 10); } - vector.getMutator().setValueCount(10); + vector.setValueCount(10); UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -119,7 +118,7 @@ public void testNestedInList() { ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); - NullableIntVector.Mutator innerMutator = (NullableIntVector.Mutator) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector().getMutator(); + NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { @@ -127,8 +126,8 @@ public void testNestedInList() { int position = mutator.startNewValue(i); for (int j = 0; j < i % 7; j++) { tupleMutator.setNotNull(position + j); - innerMutator.set((position + j) * 2, j); - innerMutator.set((position + j) * 2 + 1, j + 1); + innerVector.set((position + j) * 2, j); + innerVector.set((position + j) * 2 + 1, j + 1); } mutator.endValue(i, i % 7); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 66e5375e3bd..7de3bcb3319 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -25,7 +25,6 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.NullableVarCharVector; -import org.apache.arrow.vector.NullableVarCharVector.Accessor; import org.apache.arrow.vector.util.TransferPair; import org.junit.After; @@ -54,32 +53,30 @@ public void test() throws Exception { final int valueCount = 500; final String[] compareArray = new String[valueCount]; - final NullableVarCharVector.Mutator mutator = varCharVector.getMutator(); for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); - mutator.set(i, s.getBytes()); + varCharVector.set(i, s.getBytes()); compareArray[i] = s; } - mutator.setValueCount(valueCount); + varCharVector.setValueCount(valueCount); final TransferPair tp = varCharVector.getTransferPair(allocator); final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo(); - final Accessor accessor = newVarCharVector.getAccessor(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}}; for (final int[] startLength : startLengths) { final int start = startLength[0]; final int length = startLength[1]; tp.splitAndTransfer(start, length); - newVarCharVector.getMutator().setValueCount(length); + newVarCharVector.setValueCount(length); for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0; if (expectedSet) { final byte[] expectedValue = compareArray[start + i].getBytes(); - assertFalse(accessor.isNull(i)); - assertArrayEquals(expectedValue, accessor.get(i)); + assertFalse(newVarCharVector.isNull(i)); + assertArrayEquals(expectedValue, newVarCharVector.get(i)); } else { - assertTrue(accessor.isNull(i)); + assertTrue(newVarCharVector.isNull(i)); } } newVarCharVector.clear(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index a239861d9b3..b7f88c3babf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -694,8 +694,6 @@ public void testNullableFixedType2() { public void testNullableFixedType3() { // Create a new value vector for 1024 integers try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { - final NullableIntVector.Mutator mutator = vector.getMutator(); - final NullableIntVector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 1024; @@ -706,26 +704,26 @@ public void testNullableFixedType3() { /* underlying buffer should be able to store 16 values */ assertEquals(initialCapacity, vector.getValueCapacity()); - mutator.set(0, 1); - mutator.set(1, 2); - mutator.set(100, 3); - mutator.set(1022, 4); - mutator.set(1023, 5); + vector.set(0, 1); + vector.set(1, 2); + vector.set(100, 3); + vector.set(1022, 4); + vector.set(1023, 5); /* check vector contents */ int j = 1; for(int i = 0; i <= 1023; i++) { if((i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } else { - assertFalse("null data not expected at index: " + i, accessor.isNull(i)); - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertFalse("null data not expected at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, j, vector.get(i)); j++; } } - mutator.setValueCount(1024); + vector.setValueCount(1024); Field field = vector.getField(); TypeLayout typeLayout = field.getTypeLayout(); @@ -749,7 +747,7 @@ public void testNullableFixedType3() { assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined /* this should trigger a realloc() */ - mutator.setSafe(1024, 6); + vector.setSafe(1024, 6); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); @@ -758,11 +756,11 @@ public void testNullableFixedType3() { j = 1; for(int i = 0; i < (initialCapacity * 2); i++) { if((i > 1024) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } else { - assertFalse("null data not expected at index: " + i, accessor.isNull(i)); - assertEquals("unexpected value at index: " + i, j, accessor.get(i)); + assertFalse("null data not expected at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, j, vector.get(i)); j++; } } @@ -775,13 +773,101 @@ public void testNullableFixedType3() { /* vector data should have been zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } vector.allocateNew(4096); // vector has been erased for(int i = 0; i < 4096; i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + } + } + } + + @Test /* NullableIntVector */ + public void testNullableFixedType4() { + try (final NullableIntVector vector = newVector(NullableIntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { + + /* no memory allocation has happened yet */ + assertEquals(0, vector.getValueCapacity()); + + vector.allocateNew(); + int valueCapacity = vector.getValueCapacity(); + assertEquals(vector.INITIAL_VALUE_ALLOCATION, valueCapacity); + + int baseValue = 20000; + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + vector.set(i, baseValue + i); + } + } + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.setSafe(valueCapacity, 20000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i == valueCapacity) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, 20000000, vector.get(i)); + } else if (i < valueCapacity) { + if ((i & 1) == 1) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.zeroVector(); + + for (int i = 0; i < vector.getValueCapacity(); i+=2) { + vector.set(i, baseValue + i); + } + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i%2 == 0) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + vector.setSafe((valueCapacity * 2) + 1000, 400000000); + assertEquals(valueCapacity * 4, vector.getValueCapacity()); + + for (int i = 0; i < vector.getValueCapacity(); i++) { + if (i == (valueCapacity*2 + 1000)) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, 400000000, vector.get(i)); + } else if (i < valueCapacity*2 && (i%2) == 0) { + assertFalse("unexpected null value at index: " + i, vector.isNull(i)); + assertEquals("unexpected value at index: " + i, baseValue + i, vector.get(i)); + } else { + assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + } + } + + /* reset the vector */ + vector.reset(); + + /* capacity shouldn't change after reset */ + assertEquals(valueCapacity * 4, vector.getValueCapacity()); + + /* vector data should be zeroed out */ + for(int i = 0; i < (valueCapacity * 4); i++) { + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -805,32 +891,30 @@ public void testNullableVarType1() { // Create a new value vector for 1024 integers. try (final NullableVarCharVector vector = newNullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); - m.set(0, STR1); - m.set(1, STR2); - m.set(2, STR3); - m.setSafe(3, STR3, 1, STR3.length - 1); - m.setSafe(4, STR3, 2, STR3.length - 2); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.setSafe(3, STR3, 1, STR3.length - 1); + vector.setSafe(4, STR3, 2, STR3.length - 2); ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3); - m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); - m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); + vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); + vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); // Check the sample strings. - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6)); // Ensure null value throws. boolean b = false; try { - vector.getAccessor().get(7); + vector.get(7); } catch (IllegalStateException e) { b = true; } finally { @@ -1070,43 +1154,42 @@ public void testReallocAfterVectorTransfer2() { @Test /* NullableVarCharVector */ public void testReallocAfterVectorTransfer3() { try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - /* 4096 values with 10 byte per record */ vector.allocateNew(4096 * 10, 4096); int valueCapacity = vector.getValueCapacity(); + assertEquals(4096, valueCapacity); /* populate the vector */ for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } /* Check the vector output */ for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } /* trigger first realloc */ - mutator.setSafe(valueCapacity, STR2, 0, STR2.length); + vector.setSafe(valueCapacity, STR2, 0, STR2.length); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); /* populate the remaining vector */ for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } @@ -1114,23 +1197,24 @@ public void testReallocAfterVectorTransfer3() { valueCapacity = vector.getValueCapacity(); for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } /* trigger second realloc */ - mutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + vector.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); /* populate the remaining vector */ for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { if ((i & 1) == 1) { - mutator.set(i, STR1); + vector.set(i, STR1); } else { - mutator.set(i, STR2); + vector.set(i, STR2); } } @@ -1138,10 +1222,10 @@ public void testReallocAfterVectorTransfer3() { valueCapacity = vector.getValueCapacity(); for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertArrayEquals(STR1, accessor.get(i)); + assertArrayEquals(STR1, vector.get(i)); } else { - assertArrayEquals(STR2, accessor.get(i)); + assertArrayEquals(STR2, vector.get(i)); } } @@ -1152,13 +1236,112 @@ public void testReallocAfterVectorTransfer3() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); NullableVarCharVector toVector = (NullableVarCharVector)transferPair.getTo(); - NullableVarCharVector.Mutator toMutator = toVector.getMutator(); - NullableVarCharVector.Accessor toAccessor = toVector.getAccessor(); - valueCapacity = toVector.getValueCapacity(); - /* trigger a realloc of this toVector */ - toMutator.setSafe(valueCapacity + 10, STR2, 0, STR2.length); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, toVector.get(i)); + } + else { + assertArrayEquals(STR2, toVector.get(i)); + } + } + + toVector.close(); + } + } + + @Test /* NullableIntVector */ + public void testReallocAfterVectorTransfer4() { + try (final NullableIntVector vector = new NullableIntVector(EMPTY_SCHEMA_PATH, allocator)) { + + /* 4096 values */ + vector.allocateNew(4096); + int valueCapacity = vector.getValueCapacity(); + assertEquals(4096, valueCapacity); + + /* populate the vector */ + int baseValue = 1000; + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* trigger first realloc */ + vector.setSafe(valueCapacity, 10000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* trigger second realloc */ + vector.setSafe(valueCapacity, 10000000); + assertEquals(valueCapacity * 2, vector.getValueCapacity()); + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 0) { + vector.set(i, 1000 + i); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, vector.get(i)); + } + else { + assertTrue(vector.isNull(i)); + } + } + + /* we are potentially working with 4x the size of vector buffer + * that we initially started with. Now let's transfer the vector. + */ + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.transfer(); + NullableIntVector toVector = (NullableIntVector)transferPair.getTo(); + /* value capacity of source and target vectors should be same after + * the transfer. + */ + assertEquals(valueCapacity, toVector.getValueCapacity()); + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 0) { + assertEquals(1000 + i, toVector.get(i)); + } + else { + assertTrue(toVector.isNull(i)); + } + } toVector.close(); } @@ -1199,31 +1382,33 @@ public void testReAllocNullableFixedWidthVector() { @Test public void testReAllocNullableVariableWidthVector() { - // Create a new value vector for 1024 integers try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.allocateNew(); int initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - // Put values in indexes that fall within the initial allocation - m.setSafe(0, STR1, 0, STR1.length); - m.setSafe(initialCapacity - 1, STR2, 0, STR2.length); + /* Put values in indexes that fall within the initial allocation */ + vector.setSafe(0, STR1, 0, STR1.length); + vector.setSafe(initialCapacity - 1, STR2, 0, STR2.length); - // Now try to put values in space that falls beyond the initial allocation - m.setSafe(initialCapacity + 200, STR3, 0, STR3.length); + /* the above set calls should NOT have triggered a realloc */ + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - // Check valueCapacity is more than initial allocation - assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity()); + /* Now try to put values in space that falls beyond the initial allocation */ + vector.setSafe(initialCapacity + 200, STR3, 0, STR3.length); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(initialCapacity - 1)); - assertArrayEquals(STR3, accessor.get(initialCapacity + 200)); + /* Check valueCapacity is more than initial allocation */ + assertEquals(((initialCapacity + 1) * 2) - 1, vector.getValueCapacity()); + + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(initialCapacity - 1)); + assertArrayEquals(STR3, vector.get(initialCapacity + 200)); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when the current batch is processed. - m.setValueCount(vector.getValueCapacity() + 200); + vector.setValueCount(vector.getValueCapacity() + 200); } } @@ -1232,10 +1417,18 @@ public void testFillEmptiesNotOverfill() { try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { vector.allocateNew(); - vector.getMutator().setSafe(4094, "hello".getBytes(), 0, 5); - vector.getMutator().setValueCount(4095); + int initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); + + vector.setSafe(4094, "hello".getBytes(), 0, 5); + /* the above set method should NOT have trigerred a realloc */ + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); - assertEquals(4096 * 4, vector.getFieldBuffers().get(1).capacity()); + vector.setValueCount(4095); + assertEquals(4096 * vector.OFFSET_WIDTH, vector.getFieldBuffers().get(1).capacity()); + initialCapacity = vector.getValueCapacity(); + assertEquals(4095, initialCapacity); } } @@ -1243,42 +1436,129 @@ public void testFillEmptiesNotOverfill() { public void testCopyFromWithNulls() { try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); for (int i = 0; i < 4095; i++) { if (i % 3 == 0) { continue; } byte[] b = Integer.toString(i).getBytes(); - vector.getMutator().setSafe(i, b, 0, b.length); + vector.setSafe(i, b, 0, b.length); } - vector.getMutator().setValueCount(4095); + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } vector2.allocateNew(); + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); for (int i = 0; i < 4095; i++) { vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } } - vector2.getMutator().setValueCount(4095); + /* NO reAlloc() should have happened in copyFrom */ + capacity = vector2.getValueCapacity(); + assertEquals(4095, capacity); + + vector2.setValueCount(4095); for (int i = 0; i < 4095; i++) { if (i % 3 == 0) { - assertNull(vector2.getAccessor().getObject(i)); + assertNull(vector2.getObject(i)); } else { - assertEquals(Integer.toString(i), vector2.getAccessor().getObject(i).toString()); + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); } } } } @Test - public void testSetLastSetUsage() { - try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { + public void testCopyFromWithNulls1() { + try (final NullableVarCharVector vector = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator); + final NullableVarCharVector vector2 = newVector(NullableVarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + continue; + } + byte[] b = Integer.toString(i).getBytes(); + vector.setSafe(i, b, 0, b.length); + } + + /* NO reAlloc() should have happened in setSafe() */ + capacity = vector.getValueCapacity(); + assertEquals(4095, capacity); + + vector.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + } + } + + /* set lesser initial capacity than actually needed + * to trigger reallocs in copyFromSafe() + */ + vector2.allocateNew(1024 * 10, 1024); + + capacity = vector2.getValueCapacity(); + assertEquals(1024, capacity); + + for (int i = 0; i < 4095; i++) { + vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + + /* 2 reAllocs should have happened in copyFromSafe() */ + capacity = vector2.getValueCapacity(); + assertEquals(4096, capacity); + + vector2.setValueCount(4095); + + for (int i = 0; i < 4095; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else { + assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + } + } + } + } + + @Test + public void testSetLastSetUsage() { + try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); setBytes(0, STR1, vector); @@ -1289,32 +1569,102 @@ public void testSetLastSetUsage() { setBytes(5, STR6, vector); /* Check current lastSet */ - assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); /* * If we don't do setLastSe(5) before setValueCount(), then the latter will corrupt * the value vector by filling in all positions [0,valuecount-1] will empty byte arrays. * Run the test by commenting out next line and we should see incorrect vector output. */ - mutator.setLastSet(5); - mutator.setValueCount(20); + vector.setLastSet(5); + vector.setValueCount(20); + + /* Check current lastSet */ + assertEquals(Integer.toString(19), Integer.toString(vector.getLastSet())); /* Check the vector output again */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(10))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(11))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(15))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(16))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(17))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(18))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(19))); + + /* Check offsets */ + assertEquals(Integer.toString(0), + Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(6), + Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(16), + Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(21), + Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(30), + Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(34), + Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(16 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(17 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(18 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH))); + + vector.set(19, STR6); + assertArrayEquals(STR6, vector.get(19)); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(19 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(46), + Integer.toString(vector.offsetBuffer.getInt(20 * vector.OFFSET_WIDTH))); } } @@ -1322,29 +1672,25 @@ public void testSetLastSetUsage() { public void testVectorLoadUnload() { try (final NullableVarCharVector vector1 = new NullableVarCharVector("myvector", allocator)) { - - final NullableVarCharVector.Mutator mutator1 = vector1.getMutator(); - vector1.allocateNew(1024 * 10, 1024); - mutator1.set(0, STR1); - mutator1.set(1, STR2); - mutator1.set(2, STR3); - mutator1.set(3, STR4); - mutator1.set(4, STR5); - mutator1.set(5, STR6); - assertEquals(Integer.toString(5), Integer.toString(mutator1.getLastSet())); - mutator1.setValueCount(15); - assertEquals(Integer.toString(14), Integer.toString(mutator1.getLastSet())); + vector1.set(0, STR1); + vector1.set(1, STR2); + vector1.set(2, STR3); + vector1.set(3, STR4); + vector1.set(4, STR5); + vector1.set(5, STR6); + assertEquals(Integer.toString(5), Integer.toString(vector1.getLastSet())); + vector1.setValueCount(15); + assertEquals(Integer.toString(14), Integer.toString(vector1.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor(); - assertArrayEquals(STR1, accessor1.get(0)); - assertArrayEquals(STR2, accessor1.get(1)); - assertArrayEquals(STR3, accessor1.get(2)); - assertArrayEquals(STR4, accessor1.get(3)); - assertArrayEquals(STR5, accessor1.get(4)); - assertArrayEquals(STR6, accessor1.get(5)); + assertArrayEquals(STR1, vector1.get(0)); + assertArrayEquals(STR2, vector1.get(1)); + assertArrayEquals(STR3, vector1.get(2)); + assertArrayEquals(STR4, vector1.get(3)); + assertArrayEquals(STR5, vector1.get(4)); + assertArrayEquals(STR6, vector1.get(5)); Field field = vector1.getField(); String fieldName = field.getName(); @@ -1357,7 +1703,7 @@ public void testVectorLoadUnload() { Schema schema = new Schema(fields); - VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, accessor1.getValueCount()); + VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, vector1.getValueCount()); VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); try ( @@ -1370,24 +1716,21 @@ public void testVectorLoadUnload() { vectorLoader.load(recordBatch); NullableVarCharVector vector2 = (NullableVarCharVector) schemaRoot2.getVector(fieldName); - NullableVarCharVector.Mutator mutator2 = vector2.getMutator(); - /* * lastSet would have internally been set by VectorLoader.load() when it invokes * loadFieldBuffers. */ - assertEquals(Integer.toString(14), Integer.toString(mutator2.getLastSet())); - mutator2.setValueCount(25); - assertEquals(Integer.toString(24), Integer.toString(mutator2.getLastSet())); + assertEquals(Integer.toString(14), Integer.toString(vector2.getLastSet())); + vector2.setValueCount(25); + assertEquals(Integer.toString(24), Integer.toString(vector2.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor(); - assertArrayEquals(STR1, accessor2.get(0)); - assertArrayEquals(STR2, accessor2.get(1)); - assertArrayEquals(STR3, accessor2.get(2)); - assertArrayEquals(STR4, accessor2.get(3)); - assertArrayEquals(STR5, accessor2.get(4)); - assertArrayEquals(STR6, accessor2.get(5)); + assertArrayEquals(STR1, vector2.get(0)); + assertArrayEquals(STR2, vector2.get(1)); + assertArrayEquals(STR3, vector2.get(2)); + assertArrayEquals(STR4, vector2.get(3)); + assertArrayEquals(STR5, vector2.get(4)); + assertArrayEquals(STR6, vector2.get(5)); } } } @@ -1396,8 +1739,6 @@ public void testVectorLoadUnload() { public void testFillEmptiesUsage() { try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - vector.allocateNew(1024 * 10, 1024); setBytes(0, STR1, vector); @@ -1408,84 +1749,98 @@ public void testFillEmptiesUsage() { setBytes(5, STR6, vector); /* Check current lastSet */ - assertEquals(Integer.toString(-1), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(-1), Integer.toString(vector.getLastSet())); /* Check the vector output */ - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - - mutator.setLastSet(5); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + + vector.setLastSet(5); /* fill empty byte arrays from index [6, 9] */ - mutator.fillEmpties(10); + vector.fillEmpties(10); /* Check current lastSet */ - assertEquals(Integer.toString(9), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(9), Integer.toString(vector.getLastSet())); /* Check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9))); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); setBytes(10, STR1, vector); setBytes(11, STR2, vector); - mutator.setLastSet(11); + vector.setLastSet(11); /* fill empty byte arrays from index [12, 14] */ - mutator.setValueCount(15); + vector.setValueCount(15); /* Check current lastSet */ - assertEquals(Integer.toString(14), Integer.toString(mutator.getLastSet())); + assertEquals(Integer.toString(14), Integer.toString(vector.getLastSet())); /* Check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(6))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(7))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(8))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(9))); - assertArrayEquals(STR1, accessor.get(10)); - assertArrayEquals(STR2, accessor.get(11)); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(12))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(13))); - assertEquals(Integer.toString(0), Integer.toString(accessor.getValueLength(14))); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(6))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(7))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(8))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(9))); + assertArrayEquals(STR1, vector.get(10)); + assertArrayEquals(STR2, vector.get(11)); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(12))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(13))); + assertEquals(Integer.toString(0), Integer.toString(vector.getValueLength(14))); /* Check offsets */ - final UInt4Vector.Accessor offsetAccessor = vector.values.offsetVector.getAccessor(); - assertEquals(Integer.toString(0), Integer.toString(offsetAccessor.get(0))); - assertEquals(Integer.toString(6), Integer.toString(offsetAccessor.get(1))); - assertEquals(Integer.toString(16), Integer.toString(offsetAccessor.get(2))); - assertEquals(Integer.toString(21), Integer.toString(offsetAccessor.get(3))); - assertEquals(Integer.toString(30), Integer.toString(offsetAccessor.get(4))); - assertEquals(Integer.toString(34), Integer.toString(offsetAccessor.get(5))); - - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(6))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(7))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(8))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(9))); - assertEquals(Integer.toString(40), Integer.toString(offsetAccessor.get(10))); - - assertEquals(Integer.toString(46), Integer.toString(offsetAccessor.get(11))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(12))); - - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(13))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(14))); - assertEquals(Integer.toString(56), Integer.toString(offsetAccessor.get(15))); + assertEquals(Integer.toString(0), + Integer.toString(vector.offsetBuffer.getInt(0 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(6), + Integer.toString(vector.offsetBuffer.getInt(1 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(16), + Integer.toString(vector.offsetBuffer.getInt(2 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(21), + Integer.toString(vector.offsetBuffer.getInt(3 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(30), + Integer.toString(vector.offsetBuffer.getInt(4 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(34), + Integer.toString(vector.offsetBuffer.getInt(5 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(6 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(7 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(8 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(9 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(40), + Integer.toString(vector.offsetBuffer.getInt(10 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(46), + Integer.toString(vector.offsetBuffer.getInt(11 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(12 * vector.OFFSET_WIDTH))); + + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(13 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(14 * vector.OFFSET_WIDTH))); + assertEquals(Integer.toString(56), + Integer.toString(vector.offsetBuffer.getInt(15 * vector.OFFSET_WIDTH))); } } @@ -1493,29 +1848,25 @@ public void testFillEmptiesUsage() { public void testGetBufferAddress1() { try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator)) { - - final NullableVarCharVector.Mutator mutator = vector.getMutator(); - final NullableVarCharVector.Accessor accessor = vector.getAccessor(); - vector.allocateNew(1024 * 10, 1024); /* populate the vector */ - mutator.set(0, STR1); - mutator.set(1, STR2); - mutator.set(2, STR3); - mutator.set(3, STR4); - mutator.set(4, STR5); - mutator.set(5, STR6); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.set(3, STR4); + vector.set(4, STR5); + vector.set(5, STR6); - mutator.setValueCount(15); + vector.setValueCount(15); /* check the vector output */ - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(STR4, accessor.get(3)); - assertArrayEquals(STR5, accessor.get(4)); - assertArrayEquals(STR6, accessor.get(5)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(STR4, vector.get(3)); + assertArrayEquals(STR5, vector.get(4)); + assertArrayEquals(STR6, vector.get(5)); List buffers = vector.getFieldBuffers(); long bitAddress = vector.getValidityBufferAddress(); @@ -1531,23 +1882,18 @@ public void testGetBufferAddress1() { @Test /* NullableIntVector */ public void testGetBufferAddress2() { - try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) { - - final NullableIntVector.Mutator mutator = vector.getMutator(); - final NullableIntVector.Accessor accessor = vector.getAccessor(); boolean error = false; - vector.allocateNew(16); /* populate the vector */ for(int i = 0; i < 16; i += 2) { - mutator.set(i, i+10); + vector.set(i, i+10); } /* check the vector output */ for(int i = 0; i < 16; i += 2) { - assertEquals(i+10, accessor.get(i)); + assertEquals(i+10, vector.get(i)); } List buffers = vector.getFieldBuffers(); @@ -1580,11 +1926,15 @@ public void testMultipleClose() { vectorAllocator.close(); } + /* this method is used by the tests to bypass the vector set methods that manipulate + * lastSet. The method is to test the lastSet property and that's why we load the vector + * in a way that lastSet is not set automatically. + */ public static void setBytes(int index, byte[] bytes, NullableVarCharVector vector) { - final int currentOffset = vector.values.offsetVector.getAccessor().get(index); + final int currentOffset = vector.offsetBuffer.getInt(index * vector.OFFSET_WIDTH); - vector.bits.getMutator().setToOne(index); - vector.values.offsetVector.getMutator().set(index + 1, currentOffset + bytes.length); - vector.values.data.setBytes(currentOffset, bytes, 0, bytes.length); + BitVectorHelper.setValidityBitToOne(vector.validityBuffer, index); + vector.offsetBuffer.setInt((index + 1) * vector.OFFSET_WIDTH, currentOffset + bytes.length); + vector.valueBuffer.setBytes(currentOffset, bytes, 0, bytes.length); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 4ac7536c017..531a46c2734 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -76,24 +76,23 @@ public void testFixedType() { @Test public void testNullableType() { try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) { - final NullableVarCharVector.Mutator m = vector.getMutator(); vector.setInitialCapacity(512); vector.allocateNew(); assertEquals(512, vector.getValueCapacity()); try { - m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + vector.set(512, "foo".getBytes(StandardCharsets.UTF_8)); Assert.fail("Expected out of bounds exception"); } catch (Exception e) { // ok } vector.reAlloc(); - assertEquals(1023, vector.getValueCapacity()); + assertEquals(1024, vector.getValueCapacity()); - m.set(512, "foo".getBytes(StandardCharsets.UTF_8)); - assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8)); + vector.set(512, "foo".getBytes(StandardCharsets.UTF_8)); + assertEquals("foo", new String(vector.get(512), StandardCharsets.UTF_8)); } } @@ -105,7 +104,7 @@ public void testListType() { vector.setInitialCapacity(512); vector.allocateNew(); - assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason... + assertEquals(1023, vector.getValueCapacity()); try { vector.getOffsetVector().getAccessor().get(2014); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 7facf73f511..0b7928dca2c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -45,13 +45,21 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; public class TestVectorUnloadLoad { - static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } @Test public void testUnloadLoad() throws IOException { @@ -183,24 +191,40 @@ public void testUnloadLoadAddPadding() throws IOException { * @throws IOException */ @Test - public void testLoadEmptyValidityBuffer() throws IOException { + public void testLoadValidityBuffer() throws IOException { Schema schema = new Schema(asList( new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()), new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()) )); int count = 10; - ArrowBuf validity = allocator.buffer(10).slice(0, 0); - ArrowBuf[] values = new ArrowBuf[2]; - for (int i = 0; i < values.length; i++) { - ArrowBuf arrowBuf = allocator.buffer(count * 4); // integers - values[i] = arrowBuf; + ArrowBuf[] values = new ArrowBuf[4]; + for (int i = 0; i < 4; i+=2) { + ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0)); + ArrowBuf buf2 = allocator.buffer(count * 4); // integers + values[i] = buf1; + values[i+1] = buf2; for (int j = 0; j < count; j++) { - arrowBuf.setInt(j * 4, j); + if (i == 2) { + BitVectorHelper.setValidityBit(buf1, j, 0); + } else { + BitVectorHelper.setValidityBitToOne(buf1, j); + } + + buf2.setInt(j * 4, j); } - arrowBuf.writerIndex(count * 4); + buf1.writerIndex((int)Math.ceil(count / 8)); + buf2.writerIndex(count * 4); } + + /* + * values[0] - validity buffer for first vector + * values[1] - data buffer for first vector + * values[2] - validity buffer for second vector + * values[3] - data buffer for second vector + */ + try ( - ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(validity, values[0], validity, values[1])); + ArrowRecordBatch recordBatch = new ArrowRecordBatch(count, asList(new ArrowFieldNode(count, 0), new ArrowFieldNode(count, count)), asList(values[0], values[1], values[2], values[3])); BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("final vectors", 0, Integer.MAX_VALUE); VectorSchemaRoot newRoot = VectorSchemaRoot.create(schema, finalVectorsAllocator); ) { @@ -213,32 +237,31 @@ public void testLoadEmptyValidityBuffer() throws IOException { NullableIntVector intDefinedVector = (NullableIntVector) newRoot.getVector("intDefined"); NullableIntVector intNullVector = (NullableIntVector) newRoot.getVector("intNull"); for (int i = 0; i < count; i++) { - assertFalse("#" + i, intDefinedVector.getAccessor().isNull(i)); - assertEquals("#" + i, i, intDefinedVector.getAccessor().get(i)); - assertTrue("#" + i, intNullVector.getAccessor().isNull(i)); + assertFalse("#" + i, intDefinedVector.isNull(i)); + assertEquals("#" + i, i, intDefinedVector.get(i)); + assertTrue("#" + i, intNullVector.isNull(i)); } - intDefinedVector.getMutator().setSafe(count + 10, 1234); - assertTrue(intDefinedVector.getAccessor().isNull(count + 1)); + intDefinedVector.setSafe(count + 10, 1234); + assertTrue(intDefinedVector.isNull(count + 1)); // empty slots should still default to unset - intDefinedVector.getMutator().setSafe(count + 1, 789); - assertFalse(intDefinedVector.getAccessor().isNull(count + 1)); - assertEquals(789, intDefinedVector.getAccessor().get(count + 1)); - assertTrue(intDefinedVector.getAccessor().isNull(count)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 2)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 3)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 4)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 5)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 6)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 7)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 8)); - assertTrue(intDefinedVector.getAccessor().isNull(count + 9)); - assertFalse(intDefinedVector.getAccessor().isNull(count + 10)); - assertEquals(1234, intDefinedVector.getAccessor().get(count + 10)); + intDefinedVector.setSafe(count + 1, 789); + assertFalse(intDefinedVector.isNull(count + 1)); + assertEquals(789, intDefinedVector.get(count + 1)); + assertTrue(intDefinedVector.isNull(count)); + assertTrue(intDefinedVector.isNull(count + 2)); + assertTrue(intDefinedVector.isNull(count + 3)); + assertTrue(intDefinedVector.isNull(count + 4)); + assertTrue(intDefinedVector.isNull(count + 5)); + assertTrue(intDefinedVector.isNull(count + 6)); + assertTrue(intDefinedVector.isNull(count + 7)); + assertTrue(intDefinedVector.isNull(count + 8)); + assertTrue(intDefinedVector.isNull(count + 9)); + assertFalse(intDefinedVector.isNull(count + 10)); + assertEquals(1234, intDefinedVector.get(count + 10)); } finally { for (ArrowBuf arrowBuf : values) { arrowBuf.release(); } - validity.release(); } } @@ -258,11 +281,11 @@ public void testUnloadLoadDuplicates() throws IOException { FieldVector vector = field.createVector(originalVectorsAllocator); vector.allocateNew(); sources.add(vector); - NullableIntVector.Mutator mutator = (NullableIntVector.Mutator) vector.getMutator(); + NullableIntVector intVector = (NullableIntVector)vector; for (int i = 0; i < count; i++) { - mutator.set(i, i); + intVector.set(i, i); } - mutator.setValueCount(count); + intVector.setValueCount(count); } try (VectorSchemaRoot root = new VectorSchemaRoot(schema.getFields(), sources, count)) { @@ -277,8 +300,8 @@ public void testUnloadLoadDuplicates() throws IOException { List targets = newRoot.getFieldVectors(); Assert.assertEquals(sources.size(), targets.size()); for (int k = 0; k < sources.size(); k++) { - NullableIntVector.Accessor src = (NullableIntVector.Accessor) sources.get(k).getAccessor(); - NullableIntVector.Accessor tgt = (NullableIntVector.Accessor) targets.get(k).getAccessor(); + NullableIntVector src = (NullableIntVector) sources.get(k); + NullableIntVector tgt = (NullableIntVector) targets.get(k); Assert.assertEquals(src.getValueCount(), tgt.getValueCount()); for (int i = 0; i < count; i++) { Assert.assertEquals(src.get(i), tgt.get(i)); @@ -296,9 +319,4 @@ public static VectorUnloader newVectorUnloader(FieldVector root) { VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount); return new VectorUnloader(vsr); } - - @AfterClass - public static void afterClass() { - allocator.close(); - } } \ No newline at end of file diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index ba62de0a6d9..60009b0a463 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -97,8 +97,14 @@ public void tearDown() { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); + FieldVector fv = root.getVector("int"); + if (fv instanceof NullableIntVector) { + Assert.assertEquals(i, fv.getObject(i)); + Assert.assertEquals(Integer.valueOf(i), fv.getObject(i)); + } else { + Assert.assertEquals(i, fv.getAccessor().getObject(i)); + Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i)); + } } } @@ -152,6 +158,7 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { + Object intVal = root.getVector("int").getAccessor().getObject(i); if (i % 5 != 3) { Assert.assertEquals(i, intVal); @@ -220,22 +227,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Define dictionaries and add to provider NullableVarCharVector dictionary1Vector = newNullableVarCharVector("D1", bufferAllocator); dictionary1Vector.allocateNewSafe(); - NullableVarCharVector.Mutator mutator = dictionary1Vector.getMutator(); - mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(3); + dictionary1Vector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.set(2, "baz".getBytes(StandardCharsets.UTF_8)); + dictionary1Vector.setValueCount(3); Dictionary dictionary1 = new Dictionary(dictionary1Vector, new DictionaryEncoding(1L, false, null)); provider.put(dictionary1); NullableVarCharVector dictionary2Vector = newNullableVarCharVector("D2", bufferAllocator); dictionary2Vector.allocateNewSafe(); - mutator = dictionary2Vector.getMutator(); - mutator.set(0, "micro".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "large".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(3); + dictionary2Vector.set(0, "micro".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.set(1, "small".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.set(2, "large".getBytes(StandardCharsets.UTF_8)); + dictionary2Vector.setValueCount(3); Dictionary dictionary2 = new Dictionary(dictionary2Vector, new DictionaryEncoding(2L, false, null)); provider.put(dictionary2); @@ -243,13 +248,12 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Populate the vectors NullableVarCharVector vector1A = newNullableVarCharVector("varcharA", bufferAllocator); vector1A.allocateNewSafe(); - mutator = vector1A.getMutator(); - mutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - mutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(3, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.set(4, "bar".getBytes(StandardCharsets.UTF_8)); - mutator.set(5, "baz".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(6); + vector1A.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + vector1A.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + vector1A.set(3, "baz".getBytes(StandardCharsets.UTF_8)); + vector1A.set(4, "bar".getBytes(StandardCharsets.UTF_8)); + vector1A.set(5, "baz".getBytes(StandardCharsets.UTF_8)); + vector1A.setValueCount(6); FieldVector encodedVector1A = (FieldVector) DictionaryEncoder.encode(vector1A, dictionary1); vector1A.close(); // Done with this vector after encoding @@ -257,22 +261,20 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat // Write this vector using indices instead of encoding NullableIntVector encodedVector1B = new NullableIntVector("varcharB", bufferAllocator); encodedVector1B.allocateNewSafe(); - NullableIntVector.Mutator mutator1B = encodedVector1B.getMutator(); - mutator1B.set(0, 2); // "baz" - mutator1B.set(1, 1); // "bar" - mutator1B.set(2, 2); // "baz" - mutator1B.set(4, 1); // "bar" - mutator1B.set(5, 0); // "foo" - mutator1B.setValueCount(6); + encodedVector1B.set(0, 2); // "baz" + encodedVector1B.set(1, 1); // "bar" + encodedVector1B.set(2, 2); // "baz" + encodedVector1B.set(4, 1); // "bar" + encodedVector1B.set(5, 0); // "foo" + encodedVector1B.setValueCount(6); NullableVarCharVector vector2 = newNullableVarCharVector("sizes", bufferAllocator); vector2.allocateNewSafe(); - mutator = vector2.getMutator(); - mutator.set(1, "large".getBytes(StandardCharsets.UTF_8)); - mutator.set(2, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(3, "small".getBytes(StandardCharsets.UTF_8)); - mutator.set(4, "large".getBytes(StandardCharsets.UTF_8)); - mutator.setValueCount(6); + vector2.set(1, "large".getBytes(StandardCharsets.UTF_8)); + vector2.set(2, "small".getBytes(StandardCharsets.UTF_8)); + vector2.set(3, "small".getBytes(StandardCharsets.UTF_8)); + vector2.set(4, "large".getBytes(StandardCharsets.UTF_8)); + vector2.setValueCount(6); FieldVector encodedVector2 = (FieldVector) DictionaryEncoder.encode(vector2, dictionary2); vector2.close(); // Done with this vector after encoding @@ -355,8 +357,8 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc // Define the dictionary and add to the provider NullableVarCharVector dictionaryVector = newNullableVarCharVector("D2", bufferAllocator); dictionaryVector.allocateNewSafe(); - dictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8)); - dictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); dictionaryVector.getMutator().setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index 81e58989fcc..feae08ea223 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -581,7 +581,7 @@ public void testWriteReadFixedSizeList() throws IOException { tuples.getMutator().setNotNull(i); floats.getMutator().set(i * 2, i + 0.1f); floats.getMutator().set(i * 2 + 1, i + 10.1f); - ints.getMutator().set(i, i); + ints.set(i, i); } parent.getMutator().setValueCount(10); From 4509c29415070ec31bb62ad17b8d61a22c0e5bee Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 16 Oct 2017 00:39:30 -0700 Subject: [PATCH 02/10] ARROW-1474:[JAVA] ValueVector hierarchy (Implementation Phase 2) Implementation of all scalar types and complex types with corresponding legacy versions. Closes #1203 --- .../arrow/tools/ArrowFileTestFixtures.java | 4 +- .../apache/arrow/tools/EchoServerTest.java | 84 +- .../codegen/templates/ComplexReaders.java | 24 +- .../codegen/templates/ComplexWriters.java | 51 +- .../codegen/templates/LegacyUnionVector.java | 356 ++++++++ .../main/codegen/templates/MapWriters.java | 4 +- .../templates/NullableValueVectors.java | 787 ++---------------- .../codegen/templates/UnionListWriter.java | 11 +- .../main/codegen/templates/UnionVector.java | 212 +++-- .../main/codegen/templates/UnionWriter.java | 26 +- .../vector/BaseNullableFixedWidthVector.java | 315 +++++-- .../BaseNullableVariableWidthVector.java | 412 +++++++-- .../apache/arrow/vector/BaseValueVector.java | 33 +- .../apache/arrow/vector/BitVectorHelper.java | 115 ++- .../arrow/vector/NullableBigIntVector.java | 403 +++++++++ .../arrow/vector/NullableBitVector.java | 502 +++++++++++ .../arrow/vector/NullableDateDayVector.java | 404 +++++++++ .../arrow/vector/NullableDateMilliVector.java | 408 +++++++++ .../arrow/vector/NullableDecimalVector.java | 454 ++++++++++ .../arrow/vector/NullableFloat4Vector.java | 404 +++++++++ .../arrow/vector/NullableFloat8Vector.java | 404 +++++++++ .../arrow/vector/NullableIntVector.java | 140 +++- .../vector/NullableIntervalDayVector.java | 429 ++++++++++ .../vector/NullableIntervalYearVector.java | 387 +++++++++ .../arrow/vector/NullableSmallIntVector.java | 431 ++++++++++ .../arrow/vector/NullableTimeMicroVector.java | 404 +++++++++ .../arrow/vector/NullableTimeMilliVector.java | 405 +++++++++ .../arrow/vector/NullableTimeNanoVector.java | 402 +++++++++ .../arrow/vector/NullableTimeSecVector.java | 403 +++++++++ .../NullableTimeStampMicroTZVector.java | 219 +++++ .../vector/NullableTimeStampMicroVector.java | 220 +++++ .../NullableTimeStampMilliTZVector.java | 218 +++++ .../vector/NullableTimeStampMilliVector.java | 218 +++++ .../vector/NullableTimeStampNanoTZVector.java | 219 +++++ .../vector/NullableTimeStampNanoVector.java | 219 +++++ .../vector/NullableTimeStampSecTZVector.java | 217 +++++ .../vector/NullableTimeStampSecVector.java | 219 +++++ .../arrow/vector/NullableTimeStampVector.java | 253 ++++++ .../arrow/vector/NullableTinyIntVector.java | 432 ++++++++++ .../arrow/vector/NullableUInt1Vector.java | 318 +++++++ .../arrow/vector/NullableUInt2Vector.java | 319 +++++++ .../arrow/vector/NullableUInt4Vector.java | 291 +++++++ .../arrow/vector/NullableUInt8Vector.java | 291 +++++++ .../arrow/vector/NullableVarBinaryVector.java | 514 ++++++++++++ .../arrow/vector/NullableVarCharVector.java | 100 ++- .../org/apache/arrow/vector/ValueVector.java | 8 + .../apache/arrow/vector/VectorSchemaRoot.java | 2 +- .../apache/arrow/vector/VectorUnloader.java | 12 +- .../org/apache/arrow/vector/ZeroVector.java | 6 + .../vector/complex/AbstractMapVector.java | 4 +- .../complex/BaseRepeatedValueVector.java | 174 ++-- .../vector/complex/FixedSizeListVector.java | 238 ++++-- .../LegacyBaseRepeatedValueVector.java | 240 ++++++ .../complex/LegacyFixedSizeListVector.java | 298 +++++++ .../vector/complex/LegacyListVector.java | 288 +++++++ .../arrow/vector/complex/LegacyMapVector.java | 229 +++++ .../arrow/vector/complex/ListVector.java | 424 ++++++---- .../arrow/vector/complex/MapVector.java | 96 +-- .../vector/complex/NullableMapVector.java | 299 +++++-- .../complex/impl/NullableMapReaderImpl.java | 2 +- .../vector/complex/impl/PromotableWriter.java | 2 +- .../complex/impl/SingleMapReaderImpl.java | 2 +- .../impl/UnionFixedSizeListReader.java | 4 +- .../vector/complex/impl/UnionListReader.java | 12 +- .../vector/dictionary/DictionaryEncoder.java | 28 +- .../apache/arrow/vector/file/ArrowWriter.java | 2 +- .../vector/file/json/JsonFileReader.java | 341 ++++---- .../vector/file/json/JsonFileWriter.java | 198 +++-- .../apache/arrow/vector/util/Validator.java | 10 +- .../apache/arrow/vector/TestBitVector.java | 35 +- .../arrow/vector/TestDecimalVector.java | 12 +- .../arrow/vector/TestDictionaryVector.java | 4 +- .../arrow/vector/TestFixedSizeListVector.java | 33 +- .../apache/arrow/vector/TestListVector.java | 196 ++--- .../apache/arrow/vector/TestUnionVector.java | 269 +++--- .../apache/arrow/vector/TestValueVector.java | 222 +++-- .../arrow/vector/TestVectorReAlloc.java | 5 +- .../arrow/vector/TestVectorUnloadLoad.java | 6 +- .../complex/impl/TestPromotableWriter.java | 21 +- .../complex/writer/TestComplexWriter.java | 4 +- .../arrow/vector/file/BaseFileTest.java | 139 ++-- .../arrow/vector/file/TestArrowFile.java | 28 +- .../arrow/vector/file/TestArrowStream.java | 6 +- .../vector/file/TestArrowStreamPipe.java | 14 +- .../arrow/vector/file/json/TestJSONFile.java | 2 +- 85 files changed, 14169 insertions(+), 2427 deletions(-) create mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index 6d9a6c1323e..c56a5a3303b 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -78,8 +78,8 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E static void validateContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index ecac6d6e530..89714e44791 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -104,10 +104,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { vector.allocateNew(16); for (int j = 0; j < 8; j++) { - vector.getMutator().set(j, j + i); - vector.getMutator().set(j + 8, 0, (byte) (j + i)); + vector.set(j, j + i); + vector.set(j + 8, 0, (byte) (j + i)); } - vector.getMutator().setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); } @@ -120,10 +120,10 @@ private void testEchoServer(int serverPort, for (int i = 0; i < batches; i++) { Assert.assertTrue(reader.loadNextBatch()); assertEquals(16, reader.getVectorSchemaRoot().getRowCount()); - assertEquals(16, readVector.getAccessor().getValueCount()); + assertEquals(16, readVector.getValueCount()); for (int j = 0; j < 8; j++) { - assertEquals(j + i, readVector.getAccessor().get(j)); - assertTrue(readVector.getAccessor().isNull(j + 8)); + assertEquals(j + i, readVector.get(j)); + assertTrue(readVector.isNull(j + 8)); } } Assert.assertFalse(reader.loadNextBatch()); @@ -169,20 +169,18 @@ public void testFlatDictionary() throws IOException { FieldType.nullable(VARCHAR.getType()), allocator)) { writeVector.allocateNewSafe(); - NullableIntVector.Mutator mutator = writeVector.getMutator(); - mutator.set(0, 0); - mutator.set(1, 1); - mutator.set(3, 2); - mutator.set(4, 1); - mutator.set(5, 2); - mutator.setValueCount(6); + writeVector.set(0, 0); + writeVector.set(1, 1); + writeVector.set(3, 2); + writeVector.set(4, 1); + writeVector.set(5, 2); + writeVector.setValueCount(6); writeDictionaryVector.allocateNewSafe(); - NullableVarCharVector.Mutator dictionaryMutator = writeDictionaryVector.getMutator(); - dictionaryMutator.set(0, "foo".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.set(2, "baz".getBytes(StandardCharsets.UTF_8)); - dictionaryMutator.setValueCount(3); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(2, "baz".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(3); List fields = ImmutableList.of(writeVector.getField()); List vectors = ImmutableList.of((FieldVector) writeVector); @@ -210,23 +208,21 @@ public void testFlatDictionary() throws IOException { Assert.assertNotNull(readEncoding); Assert.assertEquals(1L, readEncoding.getId()); - FieldVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, readVector.getValueCount()); + Assert.assertEquals(0, readVector.getObject(0)); + Assert.assertEquals(1, readVector.getObject(1)); + Assert.assertEquals(null, readVector.getObject(2)); + Assert.assertEquals(2, readVector.getObject(3)); + Assert.assertEquals(1, readVector.getObject(4)); + Assert.assertEquals(2, readVector.getObject(5)); Dictionary dictionary = reader.lookup(1L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary - .getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); } } } @@ -243,9 +239,9 @@ public void testNestedDictionary() throws IOException { // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]] writeDictionaryVector.allocateNew(); - writeDictionaryVector.getMutator().set(0, "foo".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); - writeDictionaryVector.getMutator().setValueCount(2); + writeDictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); + writeDictionaryVector.setValueCount(2); writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null)); writeVector.allocateNew(); @@ -297,19 +293,17 @@ public void testNestedDictionary() throws IOException { Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new Int(32, true), encoding.getIndexType()); - ListVector.Accessor accessor = readVector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, readVector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), readVector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), readVector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), readVector.getObject(2)); Dictionary readDictionary = reader.lookup(2L); Assert.assertNotNull(readDictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) - readDictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) readDictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } } } diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index 79106490515..60347a2fcb8 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -70,11 +70,7 @@ public Field getField(){ public boolean isSet(){ <#if nullMode == "Nullable"> - <#if minor.class != "Int" && minor.class != "VarChar"> - return !vector.getAccessor().isNull(idx()); - <#else> return !vector.isNull(idx()); - <#else> return true; @@ -97,19 +93,11 @@ public void read(${minor.class?cap_first}Holder h){ public void read(Nullable${minor.class?cap_first}Holder h){ - <#if minor.class != "Int" && minor.class != "VarChar"> - vector.getAccessor().get(idx(), h); - <#else> - vector.get(idx(), h); - + vector.get(idx(), h); } public ${friendlyType} read${safeType}(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return vector.getObject(idx()); } <#if minor.class == "TimeStampSec" || @@ -118,7 +106,7 @@ public void read(Nullable${minor.class?cap_first}Holder h){ minor.class == "TimeStampNano"> @Override public ${minor.boxedType} read${minor.boxedType}(){ - return vector.getAccessor().get(idx()); + return vector.get(idx()); } @@ -127,11 +115,7 @@ public void copyValue(FieldWriter w){ } public Object readObject(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - return (Object)vector.getObject(idx()); - <#else> - return vector.getAccessor().getObject(idx()); - + return (Object)vector.getObject(idx()); } } diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 77f6594a078..406bbb39c7f 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -39,16 +39,9 @@ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Nullable${name}Vector.Mutator mutator; - - final Nullable${name}Vector vector; public ${eName}WriterImpl(Nullable${name}Vector vector) { - <#if minor.class != "Int" && minor.class != "VarChar"> - this.mutator = vector.getMutator(); - this.vector = vector; } @@ -108,51 +101,31 @@ public void setPosition(int idx) { <#else> public void write(${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write(Nullable${minor.class}Holder h) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx(), h); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx(), h); - vector.setValueCount(idx()+1); - + vector.setSafe(idx(), h); + vector.setValueCount(idx()+1); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); - vector.setValueCount(idx()+1); - + vector.setSafe(idx()<#if mode == "Nullable">, 1<#list fields as field><#if field.include!true >, ${field.name}); + vector.setValueCount(idx()+1); } - <#if minor.class == "Decimal"> + <#if minor.class == "Decimal"> public void write${minor.class}(${friendlyType} value) { - mutator.setSafe(idx(), value); - vector.getMutator().setValueCount(idx()+1); + vector.setSafe(idx(), value); + vector.setValueCount(idx()+1); } - <#if mode == "Nullable"> + <#if mode == "Nullable"> public void writeNull() { - <#if minor.class != "Int" && minor.class != "VarChar"> - mutator.setNull(idx()); - vector.getMutator().setValueCount(idx()+1); - <#else> - vector.setNull(idx()); - vector.setValueCount(idx()+1); - + vector.setNull(idx()); + vector.setValueCount(idx()+1); } diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java new file mode 100644 index 00000000000..b9cb6dfdc24 --- /dev/null +++ b/java/vector/src/main/codegen/templates/LegacyUnionVector.java @@ -0,0 +1,356 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" /> + + +<#include "/@includes/license.ftl" /> + + package org.apache.arrow.vector.complex; + +<#include "/@includes/vv_imports.ftl" /> + import com.google.common.collect.ImmutableList; + import java.util.ArrayList; + import java.util.Collections; + import java.util.Iterator; + import org.apache.arrow.vector.BaseDataValueVector; + import org.apache.arrow.vector.complex.impl.ComplexCopier; + import org.apache.arrow.vector.util.CallBack; + import org.apache.arrow.vector.schema.ArrowFieldNode; + + import static org.apache.arrow.vector.types.UnionMode.Sparse; + + + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") + + +/** + * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each + * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * as well as the addOrGet method. + * + * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup + * each time the vector is accessed. + * Source code generated using FreeMarker template ${.template_name} + */ +public class LegacyUnionVector implements FieldVector { + + private Accessor accessor = new Accessor(); + private Mutator mutator = new Mutator(); + private final UnionVector unionVector; + + public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) { + unionVector = new UnionVector(name, allocator, callBack); + } + + public BufferAllocator getAllocator() { + return unionVector.getAllocator(); + } + + @Override + public MinorType getMinorType() { + return MinorType.UNION; + } + + @Override + public void initializeChildrenFromFields(List children) { + unionVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return unionVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + unionVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return unionVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return unionVector.getFieldInnerVectors(); + } + + @Override + public long getValidityBufferAddress() { + return unionVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return unionVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } + + @Override + public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } + + public NullableMapVector getMap() { + return unionVector.getMap(); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#assign lowerCaseName = name?lower_case/> + <#if !minor.typeParams?? > + + private Nullable${name}Vector ${uncappedName}Vector; + + public Nullable${name}Vector get${name}Vector() { + return unionVector.get${name}Vector(); + } + + + + + public ListVector getList() { + return unionVector.getList(); + } + + public int getTypeValue(int index) { + return unionVector.getTypeValue(index); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + unionVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return unionVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + unionVector.reAlloc(); + } + + @Override + public void setInitialCapacity(int numRecords) { + } + + @Override + public int getValueCapacity() { + return unionVector.getValueCapacity(); + } + + @Override + public void close() { + unionVector.close(); + } + + @Override + public void clear() { + unionVector.clear(); + } + + @Override + public Field getField() { + return unionVector.getField(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return unionVector.getTransferPair(allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return unionVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return unionVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector); + } + + public void copyFrom(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { + unionVector.copyFromSafe(inIndex, outIndex, from); + } + + public FieldVector addVector(FieldVector v) { + return unionVector.addVector(v); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public FieldReader getReader() { + return unionVector.getReader(); + } + + public FieldWriter getWriter() { + return unionVector.getWriter(); + } + + @Override + public int getBufferSize() { + return unionVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return unionVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return unionVector.getBuffers(clear); + } + + @Override + public Iterator iterator() { + return unionVector.iterator(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return unionVector.getObject(index); + } + + public byte[] get(int index) { + return unionVector.get(index); + } + + public void get(int index, ComplexHolder holder) { + } + + public void get(int index, UnionHolder holder) { + unionVector.get(index, holder); + } + + public int getNullCount() { + return unionVector.getNullCount(); + } + + @Override + public int getValueCount() { + return unionVector.getValueCount(); + } + + @Override + public boolean isNull(int index) { + return unionVector.isNull(index); + } + + public int isSet(int index) { + return unionVector.isSet(index); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + UnionWriter writer; + + @Override + public void setValueCount(int valueCount) { + unionVector.setValueCount(valueCount); + } + + public void setSafe(int index, UnionHolder holder) { + unionVector.setSafe(index, holder); + } + <#list vv.types as type> + <#list type.minor as minor> + <#assign name = minor.class?cap_first /> + <#assign fields = minor.fields!type.fields /> + <#assign uncappedName = name?uncap_first/> + <#if !minor.typeParams?? > + public void setSafe(int index, Nullable${name}Holder holder) { + unionVector.setSafe(index, holder); + } + + + + + + public void setType(int index, MinorType type) { + unionVector.setType(index, type); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + @Deprecated + public int getValueCount() { return getAccessor().getValueCount(); } + + @Override + @Deprecated + public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);} + + @Override + @Deprecated + public Object getObject(int index) { return getAccessor().getObject(index); } + + @Override + @Deprecated + public int getNullCount() { return getAccessor().getNullCount(); } + + @Override + @Deprecated + public boolean isNull(int index) { return getAccessor().isNull(index); } +} diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index b89f91457e8..6afd508e688 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -184,7 +184,7 @@ public ListWriter list(String name) { } public void setValueCount(int count) { - container.getMutator().setValueCount(count); + container.setValueCount(count); } @Override @@ -199,7 +199,7 @@ public void setPosition(int index) { public void start() { <#if mode == "Single"> <#else> - container.getMutator().setIndexDefined(idx()); + container.setIndexDefined(idx()); } diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 5d1f5a33b90..8e5b10b649b 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -19,14 +19,8 @@ <#list vv.types as type> <#list type.minor as minor> -<#if minor.class == "Int" || minor.class == "VarChar"> <#assign className = "LegacyNullable${minor.class}Vector" /> <#assign valuesName = "Nullable${minor.class}Vector" /> -<#else> -<#assign className = "Nullable${minor.class}Vector" /> -<#assign valuesName = "${minor.class}Vector" /> - - <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> @@ -50,35 +44,21 @@ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -<#if minor.class == "Int" || minor.class == "VarChar"> @Deprecated - public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); protected final static byte[] emptyByteArray = new byte[]{}; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final FieldReader reader = new ${minor.class}ReaderImpl(${className}.this); - - private final String bitsField = "$bits$"; private final String valuesField = "$values$"; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final Field field; - - final BitVector bits = new BitVector(bitsField, allocator); final ${valuesName} values; private final Mutator mutator; private final Accessor accessor; - <#if minor.class != "Int" && minor.class != "VarChar"> - private final List innerVectors; - - <#if minor.typeParams??> <#assign typeParams = minor.typeParams?reverse> <#list typeParams as typeParam> @@ -122,33 +102,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj this.mutator = new Mutator(); this.accessor = new Accessor(); - <#if minor.class != "Int" && minor.class != "VarChar"> - this.field = new Field(name, fieldType, null); - innerVectors = Collections.unmodifiableList(Arrays.asList( - bits, - <#if type.major = "VarLen"> - values.offsetVector, - - values - )); - } - <#if minor.class != "Int" && minor.class != "VarChar"> - /* not needed for new vectors */ - public BitVector getValidityVector() { - return bits; - } - - @Override public List getFieldInnerVectors() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldInnerVectors(); - <#else> - return innerVectors; - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldInnerVectors(); } @Override @@ -165,123 +124,61 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.loadFieldBuffers(fieldNode, ownBuffers); - <#else> - <#if type.major = "VarLen"> - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, - values.offsetVector.getBufferSizeFor( - fieldNode.getLength() == 0? 0 : fieldNode.getLength() + 1)); - mutator.lastSet = fieldNode.getLength() - 1; - <#else> - // fixed width values truncate value vector to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, values.getBufferSizeFor(fieldNode.getLength())); - - org.apache.arrow.vector.BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - bits.valueCount = fieldNode.getLength(); - + /* DELEGATE TO NEW VECTOR */ + values.loadFieldBuffers(fieldNode, ownBuffers); } public List getFieldBuffers() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getFieldBuffers(); - <#else> - return org.apache.arrow.vector.BaseDataValueVector.unload(getFieldInnerVectors()); - + /* DELEGATE TO NEW VECTOR */ + return values.getFieldBuffers(); } @Override public Field getField() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getField(); - <#else> - return field; - + /* DELEGATE TO NEW VECTOR */ + return values.getField(); } @Override public MinorType getMinorType() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getMinorType(); - <#else> - return MinorType.${minor.class?upper_case}; - + /* DELEGATE TO NEW VECTOR */ + return values.getMinorType(); } @Override public FieldReader getReader(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getReader(); - <#else> - return reader; - + /* DELEGATE TO NEW VECTOR */ + return values.getReader(); } @Override public int getValueCapacity(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCapacity(); - <#else> - return Math.min(bits.getValueCapacity(), values.getValueCapacity()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCapacity(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBuffers(clear); - <#else> - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), ArrowBuf.class); - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - + /* DELEGATE TO NEW VECTOR */ + return values.getBuffers(clear); } @Override public void close() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.close(); - <#else> - bits.close(); - values.close(); - super.close(); - + /* DELEGATE TO NEW VECTOR */ + values.close(); } @Override public void clear() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.clear(); - <#else> - bits.clear(); - values.clear(); - super.clear(); - + /* DELEGATE TO NEW VECTOR */ + values.clear(); } @Override public int getBufferSize(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSize(); - <#else> - return values.getBufferSize() + bits.getBufferSize(); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSize(); } @Override @@ -289,14 +186,8 @@ public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSizeFor(valueCount); - <#else> - return values.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); - + /* DELEGATE TO NEW VECTOR */ + return values.getBufferSizeFor(valueCount); } public ArrowBuf getBuffer() { @@ -309,93 +200,38 @@ public ArrowBuf getBuffer() { @Override public void setInitialCapacity(int numRecords) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.setInitialCapacity(numRecords); - <#else> - bits.setInitialCapacity(numRecords); - values.setInitialCapacity(numRecords); - + /* DELEGATE TO NEW VECTOR */ + values.setInitialCapacity(numRecords); } @Override public void allocateNew() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(); - <#else> - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating buffer."); - } - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(); } @Override public boolean allocateNewSafe() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.allocateNewSafe(); - <#else> - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - success = values.allocateNewSafe() && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - return success; - + /* DELEGATE TO NEW VECTOR */ + return values.allocateNewSafe(); } @Override public void reAlloc() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reAlloc(); - <#else> - bits.reAlloc(); - values.reAlloc(); - + /* DELEGATE TO NEW VECTOR */ + values.reAlloc(); } public void reset() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.reset(); - <#else> - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.reset(); } <#if type.major == "VarLen"> @Override public void allocateNew(int totalBytes, int valueCount) { - <#if minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(totalBytes, valueCount); - <#else> - try { - values.allocateNew(totalBytes, valueCount); - bits.allocateNew(valueCount); - } catch(RuntimeException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(totalBytes, valueCount); } @Override @@ -411,21 +247,8 @@ public int getCurrentSizeInBytes(){ <#else> @Override public void allocateNew(int valueCount) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(valueCount); - <#else> - try { - values.allocateNew(valueCount); - bits.allocateNew(valueCount); - } catch(OutOfMemoryException e) { - clear(); - throw e; - } - bits.zeroVector(); - mutator.reset(); - accessor.reset(); - + /* DELEGATE TO NEW VECTOR */ + values.allocateNew(valueCount); } /** @@ -433,13 +256,8 @@ public void allocateNew(int valueCount) { */ @Override public void zeroVector() { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.zeroVector(); - <#else> - bits.zeroVector(); - values.zeroVector(); - + /* DELEGATE TO NEW VECTOR */ + values.zeroVector(); } @@ -447,53 +265,36 @@ public void zeroVector() { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator, callBack); - <#else> - return getTransferPair(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator, callBack); } @Override public TransferPair getTransferPair(BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(allocator); - <#else> - return new TransferImpl(name, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(allocator); } @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator); - <#else> - return new TransferImpl(ref, allocator); - + /* DELEGATE TO NEW VECTOR */ + return values.getTransferPair(ref, allocator); } @Override public TransferPair makeTransferPair(ValueVector to) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.makeTransferPair(to); - <#else> - return new TransferImpl((${className}) to); - + /* DELEGATE TO NEW VECTOR */ + return values.makeTransferPair(to); } - <#if minor.class == "Int" || minor.class == "VarChar"> public void transferTo(${valuesName} target) { /* DELEGATE TO NEW VECTOR */ <#if type.major == "VarLen"> @@ -512,61 +313,6 @@ public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) } - <#else> - public void transferTo(${className} target){ - bits.transferTo(target.bits); - values.transferTo(target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = mutator.lastSet; - - clear(); - } - - public void splitAndTransferTo(int startIndex, int length, ${className} target) { - bits.splitAndTransferTo(startIndex, length, target.bits); - values.splitAndTransferTo(startIndex, length, target.values); - <#if type.major == "VarLen"> - target.mutator.lastSet = length - 1; - - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - private class TransferImpl implements TransferPair { - ${className} to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new ${className}(ref, field.getFieldType(), allocator); - } - - public TransferImpl(${className} to){ - this.to = to; - } - - @Override - public ${className} getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${className}.this); - } - } - - @Override @@ -580,8 +326,6 @@ public Mutator getMutator(){ } - - <#if minor.class == "Int" || minor.class == "VarChar"> public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFrom(fromIndex, thisIndex, from); @@ -591,107 +335,43 @@ public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { /* DELEGATE TO NEW VECTOR */ values.copyFromSafe(fromIndex, thisIndex, from); } - <#else> - public void copyFrom(int fromIndex, int thisIndex, ${className} from) { - final Accessor fromAccessor = from.getAccessor(); - if (!fromAccessor.isNull(fromIndex)) { - mutator.set(thisIndex, fromAccessor.get(fromIndex)); - } - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - values.copyFromSafe(fromIndex, thisIndex, from); - bits.getMutator().setSafeToOne(thisIndex); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ - <#if type.major == "VarLen"> - mutator.fillEmpties(thisIndex); - - bits.copyFromSafe(fromIndex, thisIndex, from.bits); - values.copyFromSafe(fromIndex, thisIndex, from.values); - <#if type.major == "VarLen">mutator.lastSet = thisIndex; - } - @Override public long getValidityBufferAddress() { - /* address of the databuffer associated with the bitVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBufferAddress(); } @Override public long getDataBufferAddress() { - /* address of the dataBuffer associated with the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getDataBufferAddress(); - <#else> - return (bits.getDataBuffer().memoryAddress()); - + /* DELEGATE TO NEW VECTOR */ + return values.getDataBufferAddress(); } @Override public long getOffsetBufferAddress() { - /* address of the dataBuffer associated with the offsetVector - * this operation is not supported for fixed-width vector types. - */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBufferAddress(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetAddr()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBufferAddress(); } @Override public ArrowBuf getValidityBuffer() { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBuffer(); - <#else> - return (bits.getDataBuffer()); - + /* DELEGATE TO NEW VECTOR */ + return values.getValidityBuffer(); } @Override public ArrowBuf getDataBuffer() { - /* dataBuffer associated with the valueVector */ return (values.getDataBuffer()); } @Override public ArrowBuf getOffsetBuffer() { - /* dataBuffer associated with the offsetVector of the valueVector */ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBuffer(); - <#else> - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return (values.getOffsetBuffer()); - - + /* DELEGATE TO NEW VECTOR */ + return values.getOffsetBuffer(); } public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - final ${valuesName}.Accessor vAccessor = values.getAccessor(); /** * Get the element at the specified position. @@ -699,119 +379,67 @@ public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.m * @param index position of the value * @return value of the element, if not null */ - <#if minor.class == "Int" || minor.class == "VarChar"> - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.get(index); - } - <#else> - - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - if (isNull(index)) { - throw new IllegalStateException("Can't get a null value"); - } - return vAccessor.get(index); - } - + public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { + /* DELEGATE TO NEW VECTOR */ + return values.get(index); + } @Override public boolean isNull(int index) { - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isNull(index); - <#else> - return isSet(index) == 0; - + /* DELEGATE TO NEW VECTOR */ + return values.isNull(index); } public int isSet(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.isSet(index); - <#else> - return bAccessor.get(index); - + /* DELEGATE TO NEW VECTOR */ + return values.isSet(index); } <#if type.major == "VarLen"> public long getStartEnd(int index){ - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getStartEnd(index); - <#else> - return vAccessor.getStartEnd(index); - } @Override public int getValueLength(int index) { - <#if minor.class == "VarChar"> /* DELEGATE TO NEW VECTOR */ return values.getValueLength(index); - <#else> - return values.getAccessor().getValueLength(index); - } - <#if minor.class == "Int" || minor.class == "VarChar"> public void get(int index, Nullable${minor.class}Holder holder){ /* DELEGATE TO NEW VECTOR */ values.get(index, holder); } - <#else> - public void get(int index, Nullable${minor.class}Holder holder){ - vAccessor.get(index, holder); - holder.isSet = bAccessor.get(index); - } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public ${friendlyType} getObject(int index) { /* DELEGATE TO NEW VECTOR */ return values.getObject(index); } - <#else> - @Override - public ${friendlyType} getObject(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getObject(index); - } - } - <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> public StringBuilder getAsStringBuilder(int index) { - if (isNull(index)) { - return null; - }else{ - return vAccessor.getAsStringBuilder(index); - } + /* DELEGATE TO NEW VECTOR */ + return values.getAsStringBuilder(index); } @Override public int getValueCount(){ - <#if minor.class == "Int" || minor.class == "VarChar"> - /* DELEGATE TO NEW VECTOR */ - return values.getValueCount(); - <#else> - return bits.getAccessor().getValueCount(); - + /* DELEGATE TO NEW VECTOR */ + return values.getValueCount(); } - public void reset(){} + public void reset() { } } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { private int setCount; <#if type.major = "VarLen"> private int lastSet = -1; - private Mutator(){ - } + private Mutator() { } public ${valuesName} getVectorWithValues() { return values; @@ -819,12 +447,9 @@ private Mutator(){ @Override - public void setIndexDefined(int index){ - <#if minor.class == "Int" || minor.class == "VarChar"> + public void setIndexDefined(int index) { + /* DELEGATE TO NEW VECTOR */ values.setIndexDefined(index); - <#else> - bits.getMutator().setToOne(index); - } @@ -835,32 +460,14 @@ public void setIndexDefined(int index){ * @param index position of the bit to set * @param value array of bytes (or int if smaller than 4 bytes) to write */ - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { /* DELEGATE TO NEW VECTOR */ values.set(index, value); } - <#else> - public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { - setCount++; - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - final BitVector.Mutator bitsMutator = bits.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bitsMutator.setToOne(index); - valuesMutator.set(index, value); - <#if type.major == "VarLen">lastSet = index; - } - <#if type.major == "VarLen"> - <#if minor.class == "VarChar"> public void fillEmpties(int index) { /* DELEGATE TO NEW VECTOR */ values.fillEmpties(index); @@ -871,203 +478,55 @@ public void setValueLengthSafe(int index, int length) { /* DELEGATE TO NEW VECTOR */ values.setValueLengthSafe(index, length); } - - <#else> - public void fillEmpties(int index){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.setSafe(i, emptyByteArray); - } - while(index > bits.getValueCapacity()) { - bits.reAlloc(); - } - lastSet = index - 1; - } - - @Override - public void setValueLengthSafe(int index, int length) { - values.getMutator().setValueLengthSafe(index, length); - lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, byte[] value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, byte[] value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "VarChar"> public void setSafe(int index, ByteBuffer value, int start, int length) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value, start, length); } - <#else> - public void setSafe(int index, ByteBuffer value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value, start, length); - setCount++; - <#if type.major == "VarLen">lastSet = index; - - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setNull(int index) { /* DELEGATE TO NEW VECTOR */ values.setNull(index); } - <#else> - public void setNull(int index){ - bits.getMutator().setSafe(index, 0); - } - - - - - <#if minor.class != "Int" && minor.class != "VarChar"> - /* these methods are probably not needed */ - public void setSkipNull(int index, ${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - public void setSkipNull(int index, Nullable${minor.class}Holder holder){ - values.getMutator().set(index, holder); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, Nullable${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, Nullable${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, holder.isSet); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void set(int index, ${minor.class}Holder holder) { /* DELEGATE TO NEW VECTOR */ values.set(index, holder); } - <#else> - public void set(int index, ${minor.class}Holder holder) { - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().setToOne(index); - valuesMutator.set(index, holder); - <#if type.major == "VarLen">lastSet = index; - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public boolean isSafe(int outIndex) { /* DELEGATE TO NEW VECTOR */ return values.isSafe(outIndex); } - <#else> - public boolean isSafe(int outIndex) { - return outIndex < ${className}.this.getValueCapacity(); - } - - - <#if minor.class == "Int" || minor.class == "VarChar"> - <#if minor.class == "Int"> - public void set(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, valueField); - } - public void setSafe(int index, int isSet, int valueField) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, valueField); - } - - <#if minor.class == "VarChar"> - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, isSet, startField, endField, bufferField); - } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, isSet, startField, endField, bufferField); - } - - <#else> <#assign fields = minor.fields!type.fields /> public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ - final ${valuesName}.Mutator valuesMutator = values.getMutator(); - <#if type.major == "VarLen"> - for (int i = lastSet + 1; i < index; i++) { - valuesMutator.set(i, emptyByteArray); - } - - bits.getMutator().set(index, isSet); - valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field); - <#if type.major == "VarLen">lastSet = index; + values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, isSet); - values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field); - setCount++; - <#if type.major == "VarLen">lastSet = index; + values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); } - - - <#if minor.class == "Int" || minor.class == "VarChar"> public void setSafe(int index, Nullable${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); @@ -1077,40 +536,12 @@ public void setSafe(int index, ${minor.class}Holder value) { /* DELEGATE TO NEW VECTOR */ values.setSafe(index, value); } - <#else> - public void setSafe(int index, Nullable${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafe(index, value.isSet); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - public void setSafe(int index, ${minor.class}Holder value) { - <#if type.major == "VarLen"> - fillEmpties(index); - - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - <#if type.major == "VarLen">lastSet = index; - } - - <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { - <#if minor.class == "Int"> - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - <#else> - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; - + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } @@ -1118,93 +549,45 @@ public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if minor.class == "Decimal"> public void set(int index, ${friendlyType} value) { - bits.getMutator().setToOne(index); - values.getMutator().set(index, value); + /* DELEGATE TO NEW VECTOR */ + values.set(index, value); } public void setSafe(int index, ${friendlyType} value) { - bits.getMutator().setSafeToOne(index); - values.getMutator().setSafe(index, value); - setCount++; + /* DELEGATE TO NEW VECTOR */ + values.setSafe(index, value); } - - <#if minor.class == "Int" || minor.class == "VarChar"> @Override public void setValueCount(int valueCount) { /* DELEGATE TO NEW VECTOR */ values.setValueCount(valueCount); } - <#else> - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - <#if type.major == "VarLen"> - fillEmpties(valueCount); - - values.getMutator().setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); - } - - - <#if minor.class != "Int" && minor.class != "VarChar"> /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ @Override - public void generateTestData(int valueCount){ - bits.getMutator().generateTestDataAlt(valueCount); - values.getMutator().generateTestData(valueCount); - <#if type.major = "VarLen">lastSet = valueCount; - setValueCount(valueCount); - } - + public void generateTestData(int valueCount) { } - - <#if minor.class != "Int" && minor.class != "VarChar"> /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ @Override - public void reset(){ - setCount = 0; - <#if type.major = "VarLen">lastSet = -1; - } - - + public void reset() { } - <#if minor.class == "VarChar"> + <#if type.major == "VarLen"> public void setLastSet(int value) { /* DELEGATE TO NEW VECTOR */ values.setLastSet(value); } - <#else> - public void setLastSet(int value) { - <#if type.major = "VarLen"> - lastSet = value; - <#else> - throw new UnsupportedOperationException(); - - } - - - <#if minor.class == "VarChar"> public int getLastSet() { /* DELEGATE TO NEW VECTOR */ return values.getLastSet(); } - <#else> - public int getLastSet() { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - return lastSet; - - } } } diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index 9fe41d0d963..8ac23fe46f7 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -36,11 +36,11 @@ public class UnionListWriter extends AbstractFieldWriter { private ListVector vector; - private UInt4Vector offsets; private PromotableWriter writer; private boolean inMap = false; private String mapName; private int lastIndex = 0; + private static final int OFFSET_WIDTH = 4; public UnionListWriter(ListVector vector) { this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); @@ -49,7 +49,6 @@ public UnionListWriter(ListVector vector) { public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) { this.vector = vector; this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory); - this.offsets = vector.getOffsetVector(); } public UnionListWriter(ListVector vector, AbstractFieldWriter parent) { @@ -72,7 +71,7 @@ public Field getField() { } public void setValueCount(int count) { - vector.getMutator().setValueCount(count); + vector.setValueCount(count); } @Override @@ -133,13 +132,13 @@ public MapWriter map(String name) { @Override public void startList() { - vector.getMutator().startNewValue(idx()); - writer.setPosition(offsets.getAccessor().get(idx() + 1)); + vector.startNewValue(idx()); + writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH)); } @Override public void endList() { - offsets.getMutator().set(idx() + 1, writer.idx()); + vector.getOffsetBuffer().setInt((idx() + 1) * OFFSET_WIDTH, writer.idx()); setPosition(idx() + 1); } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 3c7ed019019..3da383fe6f3 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -25,6 +25,7 @@ <#include "/@includes/vv_imports.ftl" /> import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -32,6 +33,9 @@ import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.util.OversizedAllocationException; import static org.apache.arrow.vector.types.UnionMode.Sparse; @@ -56,12 +60,10 @@ public class UnionVector implements FieldVector { private String name; private BufferAllocator allocator; - private Accessor accessor = new Accessor(); - private Mutator mutator = new Mutator(); int valueCount; MapVector internalMap; - UInt1Vector typeVector; + ArrowBuf typeBuffer; private NullableMapVector mapVector; private ListVector listVector; @@ -71,16 +73,17 @@ public class UnionVector implements FieldVector { private int singleType = 0; private ValueVector singleVector; + private static final byte TYPE_WIDTH = 1; private final CallBack callBack; - private final List innerVectors; + private int typeBufferAllocationSizeInBytes; public UnionVector(String name, BufferAllocator allocator, CallBack callBack) { this.name = name; this.allocator = allocator; this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - this.typeVector = new UInt1Vector("types", allocator); + this.typeBuffer = allocator.getEmpty(); this.callBack = callBack; - this.innerVectors = Collections.unmodifiableList(Arrays.asList(typeVector)); + this.typeBufferAllocationSizeInBytes = BaseValueVector.INITIAL_VALUE_ALLOCATION * TYPE_WIDTH; } public BufferAllocator getAllocator() { @@ -104,20 +107,33 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // truncate types vector buffer to size (#0) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 0, typeVector.getBufferSizeFor(fieldNode.getLength())); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf buffer = ownBuffers.get(0); + typeBuffer.release(); + typeBuffer = buffer.retain(allocator); + typeBufferAllocationSizeInBytes = typeBuffer.capacity(); this.valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + typeBuffer.readerIndex(0); + typeBuffer.writerIndex(valueCount * TYPE_WIDTH); + + result.add(typeBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return this.innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use geFieldBuffers"); } private String fieldName(MinorType type) { @@ -134,7 +150,7 @@ private T addOrGet(MinorType minorType, Class c) { @Override public long getValidityBufferAddress() { - return typeVector.getDataBuffer().memoryAddress(); + return typeBuffer.memoryAddress(); } @Override @@ -148,7 +164,7 @@ public long getOffsetBufferAddress() { } @Override - public ArrowBuf getValidityBuffer() { return typeVector.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return typeBuffer; } @Override public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } @@ -211,47 +227,76 @@ public ListVector getList() { } public int getTypeValue(int index) { - return typeVector.getAccessor().get(index); - } - - public UInt1Vector getTypeVector() { - return typeVector; + return typeBuffer.getByte(index * TYPE_WIDTH); } @Override public void allocateNew() throws OutOfMemoryException { internalMap.allocateNew(); - typeVector.allocateNew(); - if (typeVector != null) { - typeVector.zeroVector(); + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + throw e; } } @Override public boolean allocateNewSafe() { boolean safe = internalMap.allocateNewSafe(); - safe = safe && typeVector.allocateNewSafe(); - if (safe) { - if (typeVector != null) { - typeVector.zeroVector(); - } + if (!safe) { return false; } + try { + allocateTypeBuffer(); + } catch (Exception e) { + clear(); + return false; } - return safe; + + return true; + } + + private void allocateTypeBuffer() { + typeBuffer = allocator.buffer(typeBufferAllocationSizeInBytes); + typeBuffer.readerIndex(0); + typeBuffer.setZero(0, typeBuffer.capacity()); } @Override public void reAlloc() { internalMap.reAlloc(); - typeVector.reAlloc(); + reallocTypeBuffer(); } - @Override - public void setInitialCapacity(int numRecords) { + private void reallocTypeBuffer() { + final int currentBufferCapacity = typeBuffer.capacity(); + long baseSize = typeBufferAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, typeBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + typeBuffer.release(1); + typeBuffer = newBuf; + typeBufferAllocationSizeInBytes = (int)newAllocationSize; } + @Override + public void setInitialCapacity(int numRecords) { } + @Override public int getValueCapacity() { - return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity()); + return Math.min(getTypeBufferValueCapacity(), internalMap.getValueCapacity()); } @Override @@ -261,7 +306,9 @@ public void close() { @Override public void clear() { - typeVector.clear(); + valueCount = 0; + typeBuffer.release(); + typeBuffer = allocator.getEmpty(); internalMap.clear(); } @@ -300,7 +347,7 @@ public TransferPair makeTransferPair(ValueVector target) { public void copyFrom(int inIndex, int outIndex, UnionVector from) { from.getReader().setPosition(inIndex); getWriter().setPosition(outIndex); - ComplexCopier.copy(from.reader, mutator.writer); + ComplexCopier.copy(from.reader, writer); } public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { @@ -321,33 +368,34 @@ public FieldVector addVector(FieldVector v) { private class TransferImpl implements TransferPair { private final TransferPair internalMapVectorTransferPair; - private final TransferPair typeVectorTransferPair; private final UnionVector to; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { to = new UnionVector(name, allocator, callBack); internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } public TransferImpl(UnionVector to) { this.to = to; internalMapVectorTransferPair = internalMap.makeTransferPair(to.internalMap); - typeVectorTransferPair = typeVector.makeTransferPair(to.typeVector); } @Override public void transfer() { + to.clear(); + to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalMapVectorTransferPair.transfer(); - typeVectorTransferPair.transfer(); to.valueCount = valueCount; } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); internalMapVectorTransferPair.splitAndTransfer(startIndex, length); - typeVectorTransferPair.splitAndTransfer(startIndex, length); - to.getMutator().setValueCount(length); + final int startPoint = startIndex * TYPE_WIDTH; + final int sliceLength = length * TYPE_WIDTH; + to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer; + to.setValueCount(length); } @Override @@ -362,13 +410,15 @@ public void copyValueSafe(int from, int to) { } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from UNION"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to UNION"); } @Override @@ -380,15 +430,17 @@ public FieldReader getReader() { } public FieldWriter getWriter() { - if (mutator.writer == null) { - mutator.writer = new UnionWriter(this); + if (writer == null) { + writer = new UnionWriter(this); } - return mutator.writer; + return writer; } @Override public int getBufferSize() { - return typeVector.getBufferSize() + internalMap.getBufferSize(); + if (valueCount == 0) { return 0; } + + return (valueCount * TYPE_WIDTH) + internalMap.getBufferSize(); } @Override @@ -402,13 +454,13 @@ public int getBufferSizeFor(final int valueCount) { bufferSize += v.getBufferSizeFor(valueCount); } - return (int) bufferSize; + return (int) bufferSize + (valueCount * TYPE_WIDTH); } @Override public ArrowBuf[] getBuffers(boolean clear) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(typeVector.getBuffers(clear)); + builder.add(typeBuffer); builder.add(internalMap.getBuffers(clear)); List list = builder.build(); return list.toArray(new ArrowBuf[list.size()]); @@ -417,15 +469,12 @@ public ArrowBuf[] getBuffers(boolean clear) { @Override public Iterator iterator() { List vectors = Lists.newArrayList(internalMap.iterator()); - vectors.add(typeVector); return vectors.iterator(); } - public class Accessor extends BaseValueVector.BaseAccessor { - @Override public Object getObject(int index) { - int type = typeVector.getAccessor().get(index); + int type = typeBuffer.getByte(index * TYPE_WIDTH); switch (MinorType.values()[type]) { case NULL: return null; @@ -436,18 +485,14 @@ public Object getObject(int index) { <#assign uncappedName = name?uncap_first/> <#if !minor.typeParams?? > case ${name?upper_case}: - <#if minor.class != "Int" && minor.class != "VarChar"> - return get${name}Vector().getAccessor().getObject(index); - <#else> return get${name}Vector().getObject(index); - case MAP: - return getMap().getAccessor().getObject(index); + return getMap().getObject(index); case LIST: - return getList().getAccessor().getObject(index); + return getList().getObject(index); default: throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]); } @@ -466,30 +511,37 @@ public void get(int index, UnionHolder holder) { holder.reader = reader; } - @Override public int getValueCount() { return valueCount; } - @Override public boolean isNull(int index) { - return typeVector.getAccessor().get(index) == 0; + return (typeBuffer.getByte(index * TYPE_WIDTH) == 0); + } + + @Override + public int getNullCount() { + int nullCount = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isNull(i)) { + nullCount++; + } + } + return nullCount; } public int isSet(int index) { return isNull(index) ? 0 : 1; } - } - - public class Mutator extends BaseValueVector.BaseMutator { UnionWriter writer; - @Override public void setValueCount(int valueCount) { - UnionVector.this.valueCount = valueCount; - typeVector.getMutator().setValueCount(valueCount); - internalMap.getMutator().setValueCount(valueCount); + this.valueCount = valueCount; + while (valueCount > getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + internalMap.setValueCount(valueCount); } public void setSafe(int index, UnionHolder holder) { @@ -534,11 +586,7 @@ public void setSafe(int index, UnionHolder holder) { <#if !minor.typeParams?? > public void setSafe(int index, Nullable${name}Holder holder) { setType(index, MinorType.${name?upper_case}); - <#if minor.class != "Int" && minor.class != "VarChar"> - get${name}Vector().getMutator().setSafe(index, holder); - <#else> - get${name}Vector().setSafe(index, holder); - + get${name}Vector().setSafe(index, holder); } @@ -546,19 +594,13 @@ public void setSafe(int index, Nullable${name}Holder holder) { public void setType(int index, MinorType type) { - typeVector.getMutator().setSafe(index, (byte) type.ordinal()); + while (index >= getTypeBufferValueCapacity()) { + reallocTypeBuffer(); + } + typeBuffer.setByte(index * TYPE_WIDTH , (byte) type.ordinal()); } - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } + private int getTypeBufferValueCapacity() { + return (int) ((typeBuffer.capacity() * 1.0) / TYPE_WIDTH); + } } diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index f892bac9133..526708a4c32 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -62,7 +62,7 @@ public void setPosition(int index) { @Override public void start() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().start(); } @@ -74,7 +74,7 @@ public void end() { @Override public void startList() { getListWriter().startList(); - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); } @Override @@ -92,7 +92,7 @@ private MapWriter getMapWriter() { } public MapWriter asMap() { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); return getMapWriter(); } @@ -106,7 +106,7 @@ private ListWriter getListWriter() { } public ListWriter asList() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); return getListWriter(); } @@ -150,19 +150,19 @@ BaseWriter getWriter(MinorType minorType) { } public ${name}Writer as${name}() { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); return get${name}Writer(); } @Override public void write(${name}Holder holder) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, ); } public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { - data.getMutator().setType(idx(), MinorType.${name?upper_case}); + data.setType(idx(), MinorType.${name?upper_case}); get${name}Writer().setPosition(idx()); get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, ); } @@ -175,28 +175,28 @@ public void writeNull() { @Override public MapWriter map() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().map(); } @Override public ListWriter list() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().list(); } @Override public ListWriter list(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().list(name); } @Override public MapWriter map(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().map(name); } @@ -209,14 +209,14 @@ public MapWriter map(String name) { <#if !minor.typeParams?? > @Override public ${capName}Writer ${lowerName}(String name) { - data.getMutator().setType(idx(), MinorType.MAP); + data.setType(idx(), MinorType.MAP); getMapWriter().setPosition(idx()); return getMapWriter().${lowerName}(name); } @Override public ${capName}Writer ${lowerName}() { - data.getMutator().setType(idx(), MinorType.LIST); + data.setType(idx(), MinorType.LIST); getListWriter().setPosition(idx()); return getListWriter().${lowerName}(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index c5f781000c9..83dc409a74f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -35,12 +35,17 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; +/** + * BaseNullableFixedWidthVector provides an abstract interface for + * implementing vectors of fixed width values. The vectors are nullable + * implying that zero or more elements in the vector could be NULL. + */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector implements FixedWidthVector, FieldVector { private final byte typeWidth; - private int valueAllocationSizeInBytes; - private int validityAllocationSizeInBytes; + protected int valueAllocationSizeInBytes; + protected int validityAllocationSizeInBytes; protected final Field field; private int allocationMonitor; @@ -52,23 +57,21 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all FieldType fieldType, final byte typeWidth) { super(name, allocator); this.typeWidth = typeWidth; - valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); field = new Field(name, fieldType, null); valueCount = 0; allocationMonitor = 0; validityBuffer = allocator.getEmpty(); valueBuffer = allocator.getEmpty(); + if (typeWidth > 0) { + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + } else { + /* specialized handling for NullableBitVector */ + valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = valueAllocationSizeInBytes; + } } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ /* TODO: * @@ -84,57 +87,91 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * See if we need logger -- unnecessary object probably - */ - - protected abstract org.slf4j.Logger getLogger(); @Override + @Deprecated public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override + @Deprecated public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return (validityBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return (valueBuffer.memoryAddress()); } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return starting address of the buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public long getOffsetBufferAddress() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + * @throws UnsupportedOperationException for fixed width vectors + */ @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException("not supported for fixed-width vectors"); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * typeWidth; @@ -142,52 +179,65 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValueBufferValueCapacity() { return (int)((valueBuffer.capacity() * 1.0)/typeWidth); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* number of bytes for the validity buffer for the given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ @Override public void zeroVector() { initValidityBuffer(); initValueBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the data buffer */ private void initValueBuffer() { valueBuffer.setZero(0, valueBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { + valueCount = 0; validityBuffer = releaseBuffer(validityBuffer); valueBuffer = releaseBuffer(valueBuffer); } @@ -208,6 +258,9 @@ protected void decrementAllocationMonitor() { allocationMonitor--; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -215,6 +268,15 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; long curAllocationSizeValidity = validityAllocationSizeInBytes; @@ -229,8 +291,7 @@ public boolean allocateNewSafe() { try{ allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -238,9 +299,20 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ public void allocateNew(int valueCount) { long valueBufferSize = valueCount * typeWidth; - long validityBufferSize = getSizeFromCount(valueCount); + long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBufferSize = validityBufferSize; + } if (allocationMonitor > 10) { /* step down the default memory allocation since we have observed @@ -266,8 +338,7 @@ public void allocateNew(int valueCount) { try { allocateBytes(valueBufferSize, validityBufferSize); } catch(Exception e) { - getLogger().error("ERROR: Failure in allocateNew"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); throw e; } @@ -294,8 +365,8 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer initValidityBuffer(); } - /* - * during splitAndTransfer, if we splitting from a random position within a byte, + /** + * During splitAndTransfer, if we splitting from a random position within a byte, * we can't just slice the source buffer so we have to explicitly allocate the * validityBuffer of the target vector. This is unlike the databuffer which we can * always slice for the target vector. @@ -307,23 +378,49 @@ private void allocateValidityBuffer(final int validityBufferSize) { initValidityBuffer(); } + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int count) { if (count == 0) { return 0; } - return (count * typeWidth) + getSizeFromCount(count); + return (count * typeWidth) + getValidityBufferSizeFromCount(count); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getSizeFromCount(valueCount); + return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[2]; @@ -338,12 +435,20 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ @Override public void reAlloc() { valueBuffer = reallocBufferHelper(valueBuffer, true); validityBuffer = reallocBufferHelper(validityBuffer, false); } + /** + * Helper method for reallocating a particular internal buffer + * Returns the new buffer. + */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (dataBuffer ? valueAllocationSizeInBytes @@ -360,10 +465,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (dataBuffer ? valueAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -381,8 +482,17 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -390,11 +500,23 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 2) { @@ -415,13 +537,22 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers validityAllocationSizeInBytes = validityBuffer.capacity(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(2); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); valueBuffer.readerIndex(0); - valueBuffer.writerIndex(valueCount * typeWidth); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } else { + valueBuffer.writerIndex(valueCount * typeWidth); + } result.add(validityBuffer); result.add(valueBuffer); @@ -429,18 +560,42 @@ public List getFieldBuffers() { return result; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableFixedWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -450,6 +605,13 @@ public void transferTo(BaseNullableFixedWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableFixedWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -459,6 +621,9 @@ public void splitAndTransferTo(int startIndex, int length, target.setValueCount(length); } + /** + * Data buffer can always be split and transferred using slicing. + */ private void splitAndTransferValueBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { final int startPoint = startIndex * typeWidth; @@ -466,17 +631,21 @@ private void splitAndTransferValueBuffer(int startIndex, int length, target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; } + /** + * Validity buffer has multiple cases of split and transfer depending on + * the starting position of the source index. + */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableFixedWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { if (offset == 0) { - // slice + /* slice */ if (target.validityBuffer != null) { target.validityBuffer.release(); } @@ -493,8 +662,10 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -509,15 +680,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -525,14 +696,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -547,27 +710,9 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } - /** * Get the value count of vector. This will always be zero unless * {@link #setValueCount(int)} has been called prior to calling this. @@ -578,7 +723,6 @@ public int getValueCount(){ return valueCount; } - /** * Set value count for the vector. * @@ -625,7 +769,6 @@ public void setValueCount(int valueCount) { } } - /** * Check if the given index is within the current value capacity * of the vector @@ -637,7 +780,6 @@ public boolean isSafe(int index) { return index < getValueCapacity(); } - /** * Check if element at given index is null. * @@ -648,7 +790,6 @@ public boolean isNull(int index) { return (isSet(index) == 0); } - /** * Same as {@link #isNull(int)}. * @@ -662,6 +803,11 @@ public int isSet(int index) { return Long.bitCount(b & (1L << bitIndex)); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -691,7 +837,6 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { ******************************************************************/ - protected void handleSafe(int index) { while (index >= getValueCapacity()) { decrementAllocationMonitor(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index a79709da4dd..6587cde633e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -45,7 +45,7 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector private int offsetAllocationSizeInBytes; /* protected members */ - protected static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ protected static final byte[] emptyByteArray = new byte[]{}; protected ArrowBuf validityBuffer; protected ArrowBuf valueBuffer; @@ -59,7 +59,7 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator FieldType fieldType) { super(name, allocator); valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; - validityAllocationSizeInBytes = getSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; field = new Field(name, fieldType, null); valueCount = 0; @@ -70,21 +70,12 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator cleared = false; } - /* TODO: - * Determine how writerIndex and readerIndex need to be used. Right now we - * are setting the writerIndex and readerIndex in the call to getFieldBuffers - * using the valueCount -- this assumes that the caller of getFieldBuffers - * on the vector has already invoked setValueCount. - * - * Do we need to set them during vector transfer and splitAndTransfer? - */ - /* TODO: * * see if getNullCount() can be made faster -- O(1) */ - /* TODO: + /* TODO: * Once the entire hierarchy has been refactored, move common functions * like getNullCount(), splitAndTransferValidityBuffer to top level * base class BaseValueVector. @@ -93,59 +84,89 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator * abstracted out to top level base class. * * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy and those vectors have not - * yet been refactored so moving things to the top class as of now - * is not a good idea. - */ - - /* TODO: - * See if we need logger -- unnecessary object probably + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. */ - /* TODO: - * Implement getBufferSize(), getCurrentSizeInBytes(). - */ - - protected abstract org.slf4j.Logger getLogger(); + @Override + @Deprecated public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not needed to write into vector"); + throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); } + @Override + @Deprecated public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not needed to read from vector"); + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ @Override public ArrowBuf getValidityBuffer() { return validityBuffer; } + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ @Override public ArrowBuf getDataBuffer() { return valueBuffer; } + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + */ @Override public ArrowBuf getOffsetBuffer() { return offsetBuffer; } + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getOffsetBufferAddress() { return offsetBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ @Override public long getValidityBufferAddress() { return validityBuffer.memoryAddress(); } + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ @Override public long getDataBufferAddress() { return valueBuffer.memoryAddress(); } + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ @Override public void setInitialCapacity(int valueCount) { final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; @@ -153,57 +174,69 @@ public void setInitialCapacity(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getSizeFromCount(valueCount); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); /* to track the end offset of last data element in vector, we need * an additional slot in offset buffer. */ offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity(){ final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); } - /* for test purposes */ private int getValidityBufferValueCapacity() { return (int)(validityBuffer.capacity() * 8L); } - /* for test purposes */ private int getOffsetBufferValueCapacity() { return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); } - /* number of bytes for the validity buffer for a given valueCount */ - protected int getSizeFromCount(int valueCount) { - return (int) Math.ceil(valueCount / 8.0); - } - + /** + * zero out the vector and the data in associated buffers. + */ public void zeroVector() { initValidityBuffer(); initOffsetBuffer(); } + /* zero out the validity buffer */ private void initValidityBuffer() { validityBuffer.setZero(0, validityBuffer.capacity()); } + /* zero out the offset buffer */ private void initOffsetBuffer() { offsetBuffer.setZero(0, offsetBuffer.capacity()); } + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ public void reset() { zeroVector(); lastSet = -1; } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clear(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { validityBuffer = releaseBuffer(validityBuffer); @@ -215,8 +248,17 @@ public void clear() { } @Override - public List getFieldInnerVectors() { throw new UnsupportedOperationException(); } + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ @Override public void initializeChildrenFromFields(List children) { if (!children.isEmpty()) { @@ -224,11 +266,24 @@ public void initializeChildrenFromFields(List children) { } } + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ @Override public List getChildrenFromFields() { return Collections.emptyList(); } + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { ArrowBuf bitBuffer = ownBuffers.get(0); @@ -246,11 +301,15 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers valueCount = fieldNode.getLength(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ public List getFieldBuffers() { List result = new ArrayList<>(3); final int lastDataOffset = getstartOffset(valueCount); validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getSizeFromCount(valueCount)); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); offsetBuffer.readerIndex(0); offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); valueBuffer.readerIndex(0); @@ -263,6 +322,9 @@ public List getFieldBuffers() { return result; } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() { if(!allocateNewSafe()){ @@ -270,6 +332,14 @@ public void allocateNew() { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int, int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ @Override public boolean allocateNewSafe() { long curAllocationSizeValue = valueAllocationSizeInBytes; @@ -287,8 +357,7 @@ public boolean allocateNewSafe() { try { allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); return false; } @@ -296,11 +365,19 @@ public boolean allocateNewSafe() { return true; } + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param totalBytes desired total memory capacity + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ @Override public void allocateNew(int totalBytes, int valueCount) { assert totalBytes >= 0; final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); if (totalBytes > MAX_ALLOCATION_SIZE || offsetBufferSize > MAX_ALLOCATION_SIZE) { @@ -313,12 +390,12 @@ public void allocateNew(int totalBytes, int valueCount) { try { allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); } catch (Exception e) { - getLogger().error("ERROR: Failure in allocateNewSafe"); - getLogger().error(e.getMessage()); + e.printStackTrace(); clear(); } } + /* allocate the inner buffers */ private void allocateBytes(final long valueBufferSize, final long validityBufferSize, final long offsetBufferSize) { /* allocate data buffer */ @@ -330,6 +407,7 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer allocateOffsetBuffer(offsetBufferSize); } + /* allocate offset buffer */ private void allocateOffsetBuffer(final long size) { final int curSize = (int)size; offsetBuffer = allocator.buffer(curSize); @@ -338,6 +416,7 @@ private void allocateOffsetBuffer(final long size) { initOffsetBuffer(); } + /* allocate validity buffer */ private void allocateValidityBuffer(final long size) { final int curSize = (int)size; validityBuffer = allocator.buffer(curSize); @@ -346,6 +425,10 @@ private void allocateValidityBuffer(final long size) { initValidityBuffer(); } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ public void reAlloc() { reallocValueBuffer(); reallocValidityAndOffsetBuffers(); @@ -378,10 +461,7 @@ protected void reallocValidityAndOffsetBuffers() { validityBuffer = reallocBufferHelper(validityBuffer, false); } - /* need to refactor this to keep the logic in an single place and make callers - * more intelligent. see handleSafe() for more comments on realloc - */ - + /* helper method to realloc a particular buffer. returns the allocated buffer */ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { final int currentBufferCapacity = buffer.capacity(); long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes @@ -398,10 +478,6 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer throw new OversizedAllocationException("Unable to expand the buffer"); } - getLogger().debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", - name, (offsetBuffer ? offsetAllocationSizeInBytes : validityAllocationSizeInBytes), - newAllocationSize); - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); newBuf.setBytes(0, buffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; @@ -418,6 +494,10 @@ private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer return buffer; } + /** + * Get the size (number of bytes) of underlying data buffer. + * @return + */ @Override public int getByteCapacity(){ return valueBuffer.capacity(); @@ -429,30 +509,55 @@ public int getCurrentSizeInBytes(){ return 0; } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { - /* TODO */ - return 0; + return getBufferSizeFor(this.valueCount); } + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) { return 0; } - final int validityBufferSize = getSizeFromCount(valueCount); + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); return validityBufferSize + offsetBufferSize + dataBufferSize; } + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ @Override public Field getField() { return field; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers = new ArrowBuf[3]; @@ -468,18 +573,42 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { return getTransferPair(ref, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ @Override public TransferPair getTransferPair(BufferAllocator allocator){ return getTransferPair(name, allocator); } + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ public void transferTo(BaseNullableVariableWidthVector target){ compareTypes(target, "transferTo"); target.clear(); @@ -491,6 +620,13 @@ public void transferTo(BaseNullableVariableWidthVector target){ clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ public void splitAndTransferTo(int startIndex, int length, BaseNullableVariableWidthVector target) { compareTypes(target, "splitAndTransferTo"); @@ -502,7 +638,12 @@ public void splitAndTransferTo(int startIndex, int length, } /* - * transfer the offsets along with data + * Transfer the offsets along with data. Unlike the data buffer, we cannot simply + * slice the offset buffer for split and transfer. The reason is that offsets + * in the target vector have to be adjusted and made relative to the staring + * offset in source vector from the start index of split. This is why, we + * need to explicitly allocate the offset buffer and set the adjusted offsets + * in the target vector. */ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); @@ -517,14 +658,14 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullab } /* - * transfer the validity. + * Transfer the validity. */ private void splitAndTransferValidityBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getSizeFromCount(length); + int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; if (length > 0) { @@ -546,8 +687,8 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); target.validityBuffer.setByte(i, (b1 + b2)); } @@ -562,15 +703,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * by shifting data from the current byte. */ if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); - byte b2 = getBitsFromNextByte(this.validityBuffer, + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); } else { - byte b1 = getBitsFromCurrentByte(this.validityBuffer, + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); } @@ -578,14 +719,6 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, } } - private static byte getBitsFromCurrentByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } - - private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } - /****************************************************************** * * @@ -600,24 +733,7 @@ private static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { * @return the number of null elements. */ public int getNullCount() { - int count = 0; - final int sizeInBytes = getSizeFromCount(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; + return BitVectorHelper.getNullCount(validityBuffer, valueCount); } /** @@ -679,24 +795,55 @@ public void setValueCount(int valueCount) { lastSet = valueCount - 1; } + /** + * Create holes in the vector upto the given index (exclusive). + * Holes will be created from the current last set position in + * the vector. + * + * @param index target index + */ public void fillEmpties(int index) { handleSafe(index, emptyByteArray.length); fillHoles(index); lastSet = index - 1; } + /** + * Set the index of last non-null element in the vector. + * It is important to call this method with appropriate value + * before calling {@link #setValueCount(int)}. + * + * @param value desired index of last non-null element. + */ public void setLastSet(int value) { lastSet = value; } + /** + * Get the index of last non-null element in the vector. + * + * @return index of the last non-null element + */ public int getLastSet() { return lastSet; } + /** + * Get the starting position (offset) in the data stream for a given + * element in the vector. + * + * @param index position of the element in the vector + * @return starting offset for the element + */ public long getStartEnd(int index) { return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); } + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ public void setIndexDefined(int index) { handleSafe(index, 0); BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -761,4 +908,97 @@ protected final void handleSafe(int index, int dataLength) { reallocValueBuffer(); } } + + + /****************************************************************** + * * + * helper methods currently * + * used by JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Method used by Json Reader to explicitly set the data of the variable + * width vector elements. The method takes care of allocating the memory + * for the vector if caller hasn't done so. + * + * This method should not be used externally. + * + * @param data ArrowBuf for storing variable width elements in the vector + * @param offset offset of the element + * @param allocator memory allocator + * @param index position of the element in the vector + * @param value array of bytes for the element + * @param valueCount number of elements in the vector + * @return buffer holding the variable width data. + */ + public static ArrowBuf set(ArrowBuf data, ArrowBuf offset, + BufferAllocator allocator, int index, byte[] value, + int valueCount) { + if (data == null) { + data = allocator.buffer(INITIAL_BYTE_COUNT); + } + final int currentBufferCapacity = data.capacity(); + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + while (currentBufferCapacity < currentStartOffset + value.length) { + final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2); + newBuf.setBytes(0, data, 0, currentBufferCapacity); + data.release(); + data = newBuf; + } + data.setBytes(currentStartOffset, value, 0, value.length); + if (index == (valueCount - 1)) { + data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH)); + } + return data; + } + + /** + * Method used by Json Writer to read a variable width element from + * the variable width vector and write to Json. + * + * This method should not be used externally. + * + * @param data buffer storing the variable width vector elements + * @param offset buffer storing the offsets of variable width vector elements + * @param index position of the element in the vector + * @return array of bytes + */ + public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + final int dataLength = + offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; + final byte[] result = new byte[dataLength]; + data.getBytes(currentStartOffset, result, 0, dataLength); + return result; + } + + /** + * Method used by Json Reader to explicitly set the offsets of the variable + * width vector data. The method takes care of allocating the memory for + * offsets if the caller hasn't done so. + * + * This method should not be used externally. + * + * @param buffer ArrowBuf to store offsets for variable width elements + * @param allocator memory allocator + * @param valueCount number of elements + * @param index position of the element + * @param value offset of the element + * @return buffer holding the offsets + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * OFFSET_WIDTH); + } + buffer.setInt(index * OFFSET_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * OFFSET_WIDTH); + } + + return buffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 1db96246485..1b96782681f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -56,7 +56,6 @@ public String toString() { @Override public void clear() { - getMutator().reset(); } @Override @@ -137,10 +136,36 @@ protected ArrowBuf releaseBuffer(ArrowBuf buffer) { return buffer; } - public int getValueCount() { return 0; } + @Override + public int getValueCount() { + return getAccessor().getValueCount(); + } + + @Override + public void setValueCount(int valueCount) { + getMutator().setValueCount(valueCount); + } + + @Override + + public Object getObject(int index) { + return getAccessor().getObject(index); + } + + @Override + + public int getNullCount() { + return getAccessor().getNullCount(); + } - public void setValueCount(int valueCount) { } + @Override + public boolean isNull(int index) { + return getAccessor().isNull(index); + } - public Object getObject(int index) { return null; } + /* number of bytes for the validity buffer for the given valueCount */ + protected static int getValidityBufferSizeFromCount(final int valueCount) { + return (int) Math.ceil(valueCount / 8.0); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 2439bd26eab..69325b5ea06 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -19,24 +19,35 @@ package org.apache.arrow.vector; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; -class BitVectorHelper { +/** + * Helper class for performing generic operations on a bit vector buffer. + * External use of this class is not recommended. + */ +public class BitVectorHelper { /** * Get the index of byte corresponding to bit index in validity buffer */ - protected static int byteIndex(int absoluteBitIndex) { + public static int byteIndex(int absoluteBitIndex) { return absoluteBitIndex >> 3; } /** * Get the relative index of bit within the byte in validity buffer */ - private static int bitIndex(int absoluteBitIndex) { + public static int bitIndex(int absoluteBitIndex) { return absoluteBitIndex & 7; } - protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + /** + * Set the bit at provided index to 1. + * + * @param validityBuffer + * @param index + */ + public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -45,7 +56,14 @@ protected static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { validityBuffer.setByte(byteIndex, currentByte); } - protected static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + /** + * Set the bit at a given index to provided value (1 or 0) + * + * @param validityBuffer + * @param index + * @param value + */ + public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); @@ -57,4 +75,91 @@ protected static void setValidityBit(ArrowBuf validityBuffer, int index, int val } validityBuffer.setByte(byteIndex, currentByte); } + + /** + * Set the bit at a given index to provided value (1 or 0). Internally + * takes care of allocating the buffer if the caller didn't do so. + * + * @param validityBuffer + * @param allocator + * @param valueCount + * @param index + * @param value + * @return ArrowBuf + */ + public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (validityBuffer == null) { + validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } + setValidityBit(validityBuffer, index, value); + if (index == (valueCount - 1)) { + validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + } + + return validityBuffer; + } + + /** + * Check if a bit at a given index is set or not. + * + * @param buffer + * @param index + * @return 1 if bit is set, 0 otherwise. + */ + public static int get(final ArrowBuf buffer, int index) { + final int byteIndex = index >> 3; + final byte b = buffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Compute the size of validity buffer required to manage a given number + * of elements in a vector. + * + * @param valueCount + * @return buffer size + */ + public static int getValidityBufferSize(int valueCount) { + return ((int) Math.ceil(valueCount / 8.0)); + } + + /** + * Given a validity buffer, find the number of bits that are not set. + * This is used to compute the number of null elements in a nullable vector. + * + * @param validityBuffer + * @param valueCount + * @return number of bits not set. + */ + public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { + if (valueCount == 0) { return 0; } + int count = 0; + final int sizeInBytes = getValidityBufferSize(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { + return (byte)((data.getByte(index) & 0xFF) >>> offset); + } + + public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte)((data.getByte(index) << (8 - offset))); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java new file mode 100644 index 00000000000..ee40d708cc1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BigIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BigIntHolder; +import org.apache.arrow.vector.holders.NullableBigIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableBigIntVector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableBigIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), + allocator); + } + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new BigIntReaderImpl(NullableBigIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIGINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBigIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BigIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BigIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBigIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBigIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBigIntVector to){ + this.to = to; + } + + @Override + public NullableBigIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java new file mode 100644 index 00000000000..eceaa61b57f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -0,0 +1,502 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.BitReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.BitHolder; +import org.apache.arrow.vector.holders.NullableBitHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +import java.util.ArrayList; +import java.util.List; + +/** + * NullableBitVector implements a fixed width (1 bit) vector of + * boolean values which could be null. Each value in the vector corresponds + * to a single bit in the underlying data stream backing the vector. + */ +public class NullableBitVector extends BaseNullableFixedWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); + } + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte)0); + reader = new BitReaderImpl(NullableBitVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity(){ + return (int)(validityBuffer.capacity() * 8L); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { return 0; } + return 2 * getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(valueCount); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); + + target.setValueCount(length); + } + + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + /* slice */ + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + + destBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + + return destBuffer; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return getBit(index); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = getBit(index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean (getBit(index) != 0); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBitVector)to); + } + + private class TransferImpl implements TransferPair { + NullableBitVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableBitVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBitVector to){ + this.to = to; + } + + @Override + public NullableBitVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java new file mode 100644 index 00000000000..949287ecafe --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateDayHolder; +import org.apache.arrow.vector.holders.NullableDateDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableDateDayVector implements a fixed width (4 bytes) vector of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateDayReaderImpl(NullableDateDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateDayHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateDayVector to){ + this.to = to; + } + + @Override + public NullableDateDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java new file mode 100644 index 00000000000..a0bdccedea4 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -0,0 +1,408 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DateMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DateMilliHolder; +import org.apache.arrow.vector.holders.NullableDateMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalDateTimes; +import org.slf4j.Logger; + +/** + * NullableDateMilliVector implements a fixed width vector (8 bytes) of + * date values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDateMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDateMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateMilliVector to){ + this.to = to; + } + + @Override + public NullableDateMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java new file mode 100644 index 00000000000..8320f90830a --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -0,0 +1,454 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DecimalHolder; +import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.DecimalUtility; +import org.apache.arrow.vector.util.TransferPair; + +import java.math.BigDecimal; + +/** + * NullableDecimalVector implements a fixed width vector (16 bytes) of + * decimal values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableDecimalVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 16; + private final FieldReader reader; + + private final int precision; + private final int scale; + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, BufferAllocator allocator, + int precision, int scale) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), + allocator); + } + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType(); + reader = new DecimalReaderImpl(NullableDecimalVector.this); + this.precision = arrowType.getPrecision(); + this.scale = arrowType.getScale(); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DECIMAL; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDecimalHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.buffer = valueBuffer; + holder.precision = precision; + holder.scale = scale; + holder.start = index * TYPE_WIDTH; + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public BigDecimal getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + if (from.isSet(fromIndex) != 0) { + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, int start, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void set(int index, BigDecimal value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + DecimalUtility.checkPrecisionAndScale(value, precision, scale); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DecimalHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, ArrowBuf buffer) { + handleSafe(index); + set(index, buffer); + } + + /** + * Same as {@link #set(int, int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, start, buffer); + } + + /** + * Same as {@link #set(int, BigDecimal)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void setSafe(int index, BigDecimal value){ + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DecimalHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void set(int index, int isSet, int start, ArrowBuf buffer) { + if (isSet > 0) { + set(index, start, buffer); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles + * the case when the position of new value is beyond the current value + * capacity of the vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, isSet, start, buffer); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value as array of bytes + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte[] value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDecimalVector)to); + } + + private class TransferImpl implements TransferPair { + NullableDecimalVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, + NullableDecimalVector.this.scale); + } + + public TransferImpl(NullableDecimalVector to){ + this.to = to; + } + + @Override + public NullableDecimalVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java new file mode 100644 index 00000000000..5b28065c402 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float4Holder; +import org.apache.arrow.vector.holders.NullableFloat4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat4Vector implements a fixed width vector (4 bytes) of + * float values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float4ReaderImpl(NullableFloat4Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public float get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Float getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, float value) { + valueBuffer.setFloat(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, float value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, float)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, float value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, float value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, float)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, float value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, float value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setFloat(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static float get(final ArrowBuf buffer, final int index) { + return buffer.getFloat(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat4Vector to){ + this.to = to; + } + + @Override + public NullableFloat4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java new file mode 100644 index 00000000000..624abf2f272 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.Float8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.Float8Holder; +import org.apache.arrow.vector.holders.NullableFloat8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableFloat8Vector implements a fixed width vector (8 bytes) of + * double values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableFloat8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float8ReaderImpl(NullableFloat8Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public double get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Double getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, double value) { + valueBuffer.setDouble(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, double value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, double)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, double value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, double value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, double)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, double value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, double value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setDouble(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static double get(final ArrowBuf buffer, final int index) { + return buffer.getDouble(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableFloat8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat8Vector to){ + this.to = to; + } + + @Override + public NullableFloat8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 26b19fa7bd5..6311daf4f5c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,10 +14,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.impl.IntReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -27,39 +27,53 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.slf4j.Logger; /** - * NullableIntVector implements a fixed width vector of values which could - * be null. A validity buffer (bit vector) is maintained to track which - * elements in the vector are null. + * NullableIntVector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private static final byte TYPE_WIDTH = 4; private final FieldReader reader; + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), allocator); } + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType, TYPE_WIDTH); reader = new IntReaderImpl(NullableIntVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.INT; @@ -116,12 +130,27 @@ public Integer getObject(int index) { } } + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); } } + /** + * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { handleSafe(thisIndex); copyFrom(fromIndex, thisIndex, from); @@ -234,17 +263,80 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int valueField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { if (isSet > 0) { - set(index, valueField); + set(index, value); } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); } } - public void setSafe(int index, int isSet, int valueField ) { + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { handleSafe(index); - set(index, isSet, valueField); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); } @@ -255,11 +347,23 @@ public void setSafe(int index, int isSet, int valueField ) { ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableIntVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java new file mode 100644 index 00000000000..174b722b043 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalDayReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalDayHolder; +import org.apache.arrow.vector.holders.NullableIntervalDayHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalDayVector implements a fixed width vector (8 bytes) of + * interval (days and milliseconds) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private static final byte MILLISECOND_OFFSET = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalDayHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startIndex = index * TYPE_WIDTH; + holder.isSet = 1; + holder.days = valueBuffer.getInt(startIndex); + holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int startIndex = index * TYPE_WIDTH; + final int days = valueBuffer.getInt(startIndex); + final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + final Period p = new Period(); + return p.plusDays(days).plusMillis(milliseconds); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [days, hours, minutes, seconds, millis] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + final int startIndex = index * TYPE_WIDTH; + + final int days = valueBuffer.getInt(startIndex); + int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + + final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + + final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + + final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; + + return(new StringBuilder(). + append(days).append(dayString). + append(hours).append(":"). + append(minutes).append(":"). + append(seconds).append("."). + append(millis)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + if (from.isSet(fromIndex) != 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void set(int index, int days, int milliseconds){ + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setInt(offsetIndex, days); + valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + set(index, holder.days, holder.milliseconds); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalDayHolder holder){ + set(index, holder.days, holder.milliseconds); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void setSafe(int index, int days, int milliseconds) { + handleSafe(index); + set(index, days, milliseconds); + } + + /** + * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalDayHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void set(int index, int isSet, int days, int milliseconds) { + if (isSet > 0) { + set(index, days, milliseconds); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void setSafe(int index, int isSet, int days, int milliseconds) { + handleSafe(index); + set(index, isSet, days, milliseconds); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalDayVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalDayVector to){ + this.to = to; + } + + @Override + public NullableIntervalDayVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java new file mode 100644 index 00000000000..303286d0ed3 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.IntervalYearReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.IntervalYearHolder; +import org.apache.arrow.vector.holders.NullableIntervalYearHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.Period; + +/** + * NullableIntervalYearVector implements a fixed width (4 bytes) vector of + * interval (years and months) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALYEAR; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalYearHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int interval = get(index); + final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final Period p = new Period(); + return p.plusYears(years).plusMonths(months); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [years, months] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + }else{ + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + int value = valueBuffer.getInt(index * TYPE_WIDTH); + + final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + + return(new StringBuilder(). + append(years).append(yearString). + append(months).append(monthString)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalYearHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalYearHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalYearVector)to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalYearVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalYearVector to){ + this.to = to; + } + + @Override + public NullableIntervalYearVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java new file mode 100644 index 00000000000..c45a8d5f5bc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -0,0 +1,431 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.SmallIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.SmallIntHolder; +import org.apache.arrow.vector.holders.NullableSmallIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableSmallIntVector implements a fixed width (2 bytes) vector of + * short values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableSmallIntVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), + allocator); + } + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.SMALLINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public short get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableSmallIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Short getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + private void setValue(int index, short value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, short value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, SmallIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, short)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, short value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, SmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, SmallIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, short value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, short)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, short value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, short value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setShort(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static short get(final ArrowBuf buffer, final int index) { + return buffer.getShort(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableSmallIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableSmallIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableSmallIntVector to){ + this.to = to; + } + + @Override + public NullableSmallIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java new file mode 100644 index 00000000000..454a4ac41a6 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -0,0 +1,404 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableTimeMicroVector implements a fixed width vector (8 bytes) of + * time (microsecond resolution) values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMicroVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMicroVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMicroVector to){ + this.to = to; + } + + @Override + public NullableTimeMicroVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java new file mode 100644 index 00000000000..8540d169242 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -0,0 +1,405 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; +import org.slf4j.Logger; + +/** + * NullableTimeMilliVector implements a fixed width (4 bytes) vector of + * time (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } + org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), + org.joda.time.DateTimeZone.UTC); + return ldt; + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMilliVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMilliVector to){ + this.to = to; + } + + @Override + public NullableTimeMilliVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java new file mode 100644 index 00000000000..015226da223 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeNanoVector implements a fixed width vector (8 bytes) of + * time (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMENANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer,final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeNanoVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeNanoVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeNanoVector to){ + this.to = to; + } + + @Override + public NullableTimeNanoVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java new file mode 100644 index 00000000000..2b2375e92f1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -0,0 +1,403 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeSecHolder; +import org.apache.arrow.vector.holders.NullableTimeSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeSecVector implements a fixed width (4 bytes) vector of + * time (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeSecVector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setInt(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeSecVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTimeSecVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeSecVector to){ + this.to = to; + } + + @Override + public NullableTimeSecVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java new file mode 100644 index 00000000000..0d25eff4e02 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMicroTZVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICROTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java new file mode 100644 index 00000000000..41f7a1112f2 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMicroReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMicroHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMicroHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMicroVector implements a fixed width vector (8 bytes) of + * timestamp (microsecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMicroVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ + final long micros = get(index); + final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroVector )to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java new file mode 100644 index 00000000000..6376b085e64 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampMilliTZVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLITZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java new file mode 100644 index 00000000000..ff581d2ea2f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampMilliReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampMilliHolder; +import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampMilliVector implements a fixed width vector (8 bytes) of + * timestamp (millisecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampMilliVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = get(index); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java new file mode 100644 index 00000000000..6c4445d500b --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampNanoTZVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANOTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoTZHolder holder) { + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java new file mode 100644 index 00000000000..096913ebf1c --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampNanoReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampNanoHolder; +import org.apache.arrow.vector.holders.NullableTimeStampNanoHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampNanoVector implements a fixed width vector (8 bytes) of + * timestamp (nanosecond resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampNanoVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long nanos = get(index); + final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java new file mode 100644 index 00000000000..8e5ba72ae1f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -0,0 +1,217 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecTZReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecTZHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecTZHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampSecTZVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer + * (bit vector) is maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecTZVector extends NullableTimeStampVector { + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSECTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecTZHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecTZHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecTZHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecTZVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java new file mode 100644 index 00000000000..bc800e0e4f0 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TimeStampSecReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TimeStampSecHolder; +import org.apache.arrow.vector.holders.NullableTimeStampSecHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.joda.time.LocalDateTime; + +/** + * NullableTimeStampSecVector implements a fixed width vector (8 bytes) of + * timestamp (seconds resolution) values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTimeStampSecVector extends NullableTimeStampVector { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long secs = get(index); + final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecHolder holder){ + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecVector)to); + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java new file mode 100644 index 00000000000..b2a58bd4568 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTimeStampVector is an abstract interface for fixed width vector (8 bytes) + * of timestamp values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { + protected static final byte TYPE_WIDTH = 8; + + /** + * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + protected void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and * + * JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, long value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setLong(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + public class TransferImpl implements TransferPair { + NullableTimeStampVector to; + + public TransferImpl(NullableTimeStampVector to) { + this.to = to; + } + + @Override + public NullableTimeStampVector getTo(){ + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java new file mode 100644 index 00000000000..15100530d0e --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.TinyIntReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.TinyIntHolder; +import org.apache.arrow.vector.holders.NullableTinyIntHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableTinyIntVector implements a fixed width (1 bytes) vector of + * byte values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableTinyIntVector extends BaseNullableFixedWidthVector { + public static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), + allocator); + } + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TINYINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTinyIntHolder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TinyIntHolder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TinyIntHolder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, byte)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + + /****************************************************************** + * * + * helper routines currently * + * used in JsonFileReader and JsonFileWriter * + * * + ******************************************************************/ + + + /** + * Given a data buffer, this method sets the element value at a particular + * position. Reallocates the buffer if needed. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param allocator allocator + * @param valueCount number of elements in the vector + * @param index position of the new element + * @param value element value + * @return data buffer + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, byte value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * TYPE_WIDTH); + } + buffer.setByte(index * TYPE_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * TYPE_WIDTH); + } + + return buffer; + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static byte get(final ArrowBuf buffer, final int index) { + return buffer.getByte(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTinyIntVector)to); + } + + private class TransferImpl implements TransferPair { + NullableTinyIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTinyIntVector to){ + this.to = to; + } + + @Override + public NullableTinyIntVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java new file mode 100644 index 00000000000..4d20f800ba1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt1ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt1Holder; +import org.apache.arrow.vector.holders.NullableUInt1Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +/** + * NullableUInt1Vector implements a fixed width (1 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt1Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + public NullableUInt1Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), + allocator); + } + + public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT1; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt1Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt1Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt1Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt1Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt1Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt1Vector to){ + this.to = to; + } + + @Override + public NullableUInt1Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java new file mode 100644 index 00000000000..b140b9f5cbc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt2ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt2Holder; +import org.apache.arrow.vector.holders.NullableUInt2Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt2Vector implements a fixed width (2 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt2Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + public NullableUInt2Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), + allocator); + } + + public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT2; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public char get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt2Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Character getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + private void setValue(int index, char value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, char value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt2Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, char)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, char value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt2Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, char value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, char value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt2Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt2Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt2Vector to){ + this.to = to; + } + + @Override + public NullableUInt2Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java new file mode 100644 index 00000000000..fc8cb639581 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt4ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt4Holder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt4Vector implements a fixed width (4 bytes) vector of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt4Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableUInt4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), + allocator); + } + + public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt4Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt4Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt4Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt4Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt4Vector to){ + this.to = to; + } + + @Override + public NullableUInt4Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java new file mode 100644 index 00000000000..2110cf03b9f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.UInt8ReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.UInt8Holder; +import org.apache.arrow.vector.holders.NullableUInt8Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.slf4j.Logger; + +/** + * NullableUInt8Vector implements a fixed width vector (8 bytes) of + * integer values which could be null. A validity buffer (bit vector) is + * maintained to track which elements in the vector are null. + */ +public class NullableUInt8Vector extends BaseNullableFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + public NullableUInt8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), + allocator); + } + + public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + } + + @Override + public FieldReader getReader(){ + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt8Holder holder){ + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return get(index); + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + if(holder.isSet < 0) { + throw new IllegalArgumentException(); + } + else if(holder.isSet > 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt8Holder holder){ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt8Holder holder){ + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt8Vector)to); + } + + private class TransferImpl implements TransferPair { + NullableUInt8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt8Vector to){ + this.to = to; + } + + @Override + public NullableUInt8Vector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + } + } +} \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java new file mode 100644 index 00000000000..330e06710fc --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -0,0 +1,514 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.VarBinaryReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.VarBinaryHolder; +import org.apache.arrow.vector.holders.NullableVarBinaryHolder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +import java.nio.ByteBuffer; + +/** + * NullableVarBinaryVector implements a variable width vector of binary + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ +public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { + private final FieldReader reader; + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); + } + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader(){ + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARBINARY; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if(isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return byte array for non-null element, null otherwise + */ + public byte[] getObject(int index) { + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + return b; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarBinaryHolder holder){ + assert index >= 0; + if(isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + holder.isSet = 1; + holder.start = startOffset; + holder.end = dataLength; + holder.buffer = valueBuffer; + } + + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillHoles(thisIndex); + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + fillEmpties(thisIndex); + if (from.isSet(fromIndex) != 0) { + setSafe(thisIndex, from.get(fromIndex)); + lastSet = thisIndex; + } + } + + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + handleSafe(index, 0); + /* not really needed to set the bit to 0 as long as + * the buffer always starts from 0. + */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + handleSafe(index, end); + set(index, isSet, start, end, buffer); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator){ + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarBinaryVector)to); + } + + private class TransferImpl implements TransferPair { + NullableVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator){ + to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarBinaryVector to){ + this.to = to; + } + + @Override + public NullableVarBinaryVector getTo(){ + return to; + } + + @Override + public void transfer(){ + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index b9021544b0e..0b03164f0f3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; @@ -32,30 +31,50 @@ import java.nio.ByteBuffer; +/** + * NullableVarCharVector implements a variable width vector of VARCHAR + * values which could be NULL. A validity buffer (bit vector) is maintained + * to track which elements in the vector are null. + */ public class NullableVarCharVector extends BaseNullableVariableWidthVector { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(NullableIntVector.class); private final FieldReader reader; + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, BufferAllocator allocator) { this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); } + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator, fieldType); reader = new VarCharReaderImpl(NullableVarCharVector.this); } - @Override - protected org.slf4j.Logger getLogger() { - return logger; - } - + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ @Override public FieldReader getReader(){ return reader; } + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ @Override public Types.MinorType getMinorType() { return Types.MinorType.VARCHAR; @@ -154,7 +173,13 @@ public void get(int index, NullableVarCharHolder holder){ ******************************************************************/ - + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { fillHoles(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -163,6 +188,14 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { } } + /** + * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { fillEmpties(thisIndex); if (from.isSet(fromIndex) != 0) { @@ -383,21 +416,42 @@ public void setNull(int index){ BitVectorHelper.setValidityBit(validityBuffer, index, 0); } - public void set(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, isSet); final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + endField); - final ArrowBuf bb = bufferField.slice(startField, endField); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); + final ArrowBuf bb = buffer.slice(start, end); valueBuffer.setBytes(startOffset, bb); lastSet = index; } - public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf bufferField ) { + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { assert index >= 0; - handleSafe(index, endField); - set(index, isSet, startField, endField, bufferField); + handleSafe(index, end); + set(index, isSet, start, end, buffer); } @@ -407,11 +461,23 @@ public void setSafe(int index, int isSet, int startField, int endField, ArrowBuf * * ******************************************************************/ + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator){ return new TransferImpl(ref, allocator); } + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((NullableVarCharVector)to); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index e6048b4b517..dc74afaa2e4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -256,9 +256,17 @@ interface Mutator { */ public ArrowBuf getOffsetBuffer(); + /* temporarily add these methods here until we remove other vectors + * (non-nullable) which are under ValueVector hierarchy and still + * use the mutator/accessor interfaces. + */ public int getValueCount(); public void setValueCount(int valueCount); public Object getObject(int index); + + public int getNullCount(); + + public boolean isNull(int index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index 0c8868cad55..196393fb958 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -38,7 +38,7 @@ public class VectorSchemaRoot implements AutoCloseable { private final Map fieldVectorsMap = new HashMap<>(); public VectorSchemaRoot(FieldVector parent) { - this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getAccessor().getValueCount()); + this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount()); } public VectorSchemaRoot(List fields, List fieldVectors, int rowCount) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 96243eb5876..288afc6bbec 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -55,17 +55,7 @@ public ArrowRecordBatch getRecordBatch() { } private void appendNodes(FieldVector vector, List nodes, List buffers) { - Accessor accessor = null; - if (vector instanceof NullableIntVector) { - nodes.add(new ArrowFieldNode(((NullableIntVector)vector).getValueCount(), - includeNullCount ? ((NullableIntVector)vector).getNullCount() : -1)); - } else if (vector instanceof NullableVarCharVector) { - nodes.add(new ArrowFieldNode(((NullableVarCharVector)vector).getValueCount(), - includeNullCount ? ((NullableVarCharVector)vector).getNullCount() : -1)); - } else { - accessor = vector.getAccessor(); - nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount() : -1)); - } + nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List fieldBuffers = vector.getFieldBuffers(); List expectedBuffers = vector.getField().getTypeLayout().getVectorTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index fcef02f4a1b..3cc93a2a34f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -268,4 +268,10 @@ public void setValueCount(int valueCount) { } @Override public Object getObject(int index) { return null; } + + @Override + public int getNullCount() { return 0; } + + @Override + public boolean isNull(int index) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index 26b0f90581f..1b20b09d81b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -141,8 +141,8 @@ public T addOrGet(String childName, FieldType fieldType, } private boolean nullFilled(ValueVector vector) { - for (int r = 0; r < vector.getAccessor().getValueCount(); r++) { - if (!vector.getAccessor().isNull(r)) { + for (int r = 0; r < vector.getValueCount(); r++) { + if (!vector.isNull(r)) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 8e2877f892a..3eab30a32d2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Iterator; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BaseValueVector; @@ -31,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.SchemaChangeRuntimeException; import com.google.common.base.Preconditions; @@ -41,12 +43,14 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; public final static String DATA_VECTOR_NAME = "$data$"; - protected final UInt4Vector offsets; + public final static byte OFFSET_WIDTH = 4; + protected ArrowBuf offsetBuffer; protected FieldVector vector; protected final CallBack callBack; + protected int valueCount; + protected int offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { this(name, allocator, DEFAULT_DATA_VECTOR, callBack); @@ -54,42 +58,73 @@ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, CallBa protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { super(name, allocator); - this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.offsetBuffer = allocator.getEmpty(); this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); this.callBack = callBack; + this.valueCount = 0; } @Override public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; + boolean dataAlloc = false; try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - } finally { - if (!success) { + allocateOffsetBuffer(offsetAllocationSizeInBytes); + dataAlloc = vector.allocateNewSafe(); + } catch (Exception e) { + e.printStackTrace(); + clear(); + return false; + } + finally { + if (!dataAlloc) { clear(); } } - offsets.zeroVector(); - return success; + return dataAlloc; + } + + protected void allocateOffsetBuffer(final long size) { + final int curSize = (int)size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + offsetBuffer.setZero(0, offsetBuffer.capacity()); } @Override public void reAlloc() { - offsets.reAlloc(); + reallocOffsetBuffer(); vector.reAlloc(); } + protected void reallocOffsetBuffer() { + final int currentBufferCapacity = offsetBuffer.capacity(); + long baseSize = offsetAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + offsetBuffer.release(1); + offsetBuffer = newBuf; + offsetAllocationSizeInBytes = (int)newAllocationSize; + } + @Override + @Deprecated public UInt4Vector getOffsetVector() { - return offsets; + throw new UnsupportedOperationException("There is no inner offset vector"); } @Override @@ -99,25 +134,29 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); + offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } @Override public int getValueCapacity() { - final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); if (vector == DEFAULT_DATA_VECTOR) { return offsetValueCapacity; } return Math.min(vector.getValueCapacity(), offsetValueCapacity); } + private int getOffsetBufferValueCapacity() { + return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + } + @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + vector.getBufferSize(); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize(); } @Override @@ -126,7 +165,7 @@ public int getBufferSizeFor(int valueCount) { return 0; } - return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount); } @Override @@ -136,14 +175,16 @@ public Iterator iterator() { @Override public void clear() { - offsets.clear(); + offsetBuffer = releaseBuffer(offsetBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, + vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -187,54 +228,61 @@ protected void replaceDataVector(FieldVector v) { vector = v; } - public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } + @Override + public int getValueCount() { + return valueCount; + } - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } + /* returns the value count for inner data vector for this list vector */ + public int getInnerValueCount() { + return vector.getValueCount(); + } - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); - } - @Override - public boolean isNull(int index) { - return false; - } + /* returns the value count for inner data vector at a particular index */ + public int getInnerValueCountAt(int index) { + return offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - + offsetBuffer.getInt(index * OFFSET_WIDTH); + } - @Override - public boolean isEmpty(int index) { - return false; - } + public boolean isNull(int index) { + return false; } - public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + public boolean isEmpty(int index) { + return false; + } - @Override - public int startNewValue(int index) { - while (offsets.getValueCapacity() <= index) { - offsets.reAlloc(); - } - int offset = offsets.getAccessor().get(index); - offsets.getMutator().setSafe(index + 1, offset); - setValueCount(index + 1); - return offset; + public int startNewValue(int index) { + while (index >= getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + int offset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, offset); + setValueCount(index + 1); + return offset; + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getOffsetBufferValueCapacity()) { + reallocOffsetBuffer(); } + final int childValueCount = valueCount == 0 ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } + + @Override + @Deprecated + public RepeatedAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from LIST."); } + @Override + @Deprecated + public RepeatedMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to LIST"); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index e95442acf6d..bb091f68c18 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -22,16 +22,14 @@ import static java.util.Collections.singletonList; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; +import java.util.*; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.*; @@ -42,10 +40,7 @@ import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.*; public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { @@ -55,15 +50,13 @@ public static FixedSizeListVector empty(String name, int size, BufferAllocator a } private FieldVector vector; - private final BitVector bits; + private ArrowBuf validityBuffer; private final int listSize; private final FieldType fieldType; - private final List innerVectors; private UnionFixedSizeListReader reader; - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private int valueCount; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -80,13 +73,14 @@ public FixedSizeListVector(String name, FieldType fieldType, CallBack schemaChangeCallback) { super(name, allocator); - this.bits = new BitVector("$bits$", allocator); + this.validityBuffer = allocator.getEmpty(); this.vector = ZeroVector.INSTANCE; this.fieldType = fieldType; this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize(); Preconditions.checkArgument(listSize > 0, "list size must be positive"); - this.innerVectors = Collections.singletonList((BufferBacked) bits); this.reader = new UnionFixedSizeListReader(this); + this.valueCount = 0; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override @@ -124,27 +118,47 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, innerVectors, ownBuffers); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(innerVectors); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); } @Override @@ -154,7 +168,9 @@ public UnionFixedSizeListReader getReader() { @Override public void allocateNew() throws OutOfMemoryException { - allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } } @Override @@ -166,31 +182,63 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = bits.allocateNewSafe() && vector.allocateNewSafe(); + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + success = vector.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - if (success) { - bits.zeroVector(); - } - return success; + + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + reallocValidityBuffer(); vector.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + public FieldVector getDataVector() { return vector; } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); vector.setInitialCapacity(numRecords * listSize); } @@ -199,15 +247,15 @@ public int getValueCapacity() { if (vector == ZeroVector.INSTANCE) { return 0; } - return vector.getValueCapacity() / listSize; + return Math.min(vector.getValueCapacity() / listSize, getValidityBufferValueCapacity()); } @Override public int getBufferSize() { - if (accessor.getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return bits.getBufferSize() + vector.getBufferSize(); + return getValidityBufferSizeFromCount(valueCount) + vector.getBufferSize(); } @Override @@ -215,7 +263,8 @@ public int getBufferSizeFor(int valueCount) { if (valueCount == 0) { return 0; } - return bits.getBufferSizeFor(valueCount) + vector.getBufferSizeFor(valueCount * listSize); + return getValidityBufferSizeFromCount(valueCount) + + vector.getBufferSizeFor(valueCount * listSize); } @Override @@ -225,14 +274,15 @@ public Iterator iterator() { @Override public void clear() { - bits.clear(); + validityBuffer = releaseBuffer(validityBuffer); vector.clear(); + valueCount = 0; super.clear(); } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -288,7 +338,7 @@ public UnionVector promoteToUnion() { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return validityBuffer.memoryAddress(); } @Override @@ -303,7 +353,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return (bits.getDataBuffer()); + return validityBuffer; } @Override @@ -316,62 +366,64 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(listSize); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - for (int i = 0; i < listSize; i++) { - vals.add(vector.getObject(index * listSize + i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vector.getAccessor(); - for (int i = 0; i < listSize; i++) { - vals.add(valuesAccessor.getObject(index * listSize + i)); - } - } - return vals; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; } - - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; + final List vals = new JsonStringArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + vals.add(vector.getObject(index * listSize + i)); } + return vals; + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } + public boolean isNull(int index) { + return (isSet(index) == 0); + } - @Override - public int getValueCount() { - return bits.getAccessor().getValueCount(); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); } - public class Mutator extends BaseValueVector.BaseMutator { + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); + @Override + public int getValueCount() { + return valueCount; + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); + public void setNotNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void setValueCount(int valueCount) { - bits.getMutator().setValueCount(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(valueCount * listSize); - } else { - vector.getMutator().setValueCount(valueCount * listSize); - } + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + while (valueCount > getValidityBufferValueCapacity()) { + reallocValidityBuffer(); } + vector.setValueCount(valueCount * listSize); } @Override @@ -392,7 +444,7 @@ public TransferPair makeTransferPair(ValueVector target) { private class TransferImpl implements TransferPair { FixedSizeListVector to; - TransferPair pairs[] = new TransferPair[2]; + TransferPair dataPair; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new FixedSizeListVector(name, allocator, fieldType, callBack)); @@ -401,19 +453,20 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(FixedSizeListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - pairs[0] = bits.makeTransferPair(to.bits); - pairs[1] = vector.makeTransferPair(to.vector); + dataPair = vector.makeTransferPair(to.vector); } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + to.clear(); + dataPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { + to.clear(); to.allocateNew(); for (int i = 0; i < length; i++) { copyValueSafe(startIndex + i, i); @@ -426,12 +479,15 @@ public ValueVector getTo() { } @Override - public void copyValueSafe(int from, int to) { - pairs[0].copyValueSafe(from, to); - int fromOffset = from * listSize; - int toOffset = to * listSize; + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= to.getValueCapacity()) { + to.reAlloc(); + } + BitVectorHelper.setValidityBit(to.validityBuffer, toIndex, isSet(fromIndex)); + int fromOffset = fromIndex * listSize; + int toOffset = toIndex * listSize; for (int i = 0; i < listSize; i++) { - pairs[1].copyValueSafe(fromOffset + i, toOffset + i); + dataPair.copyValueSafe(fromOffset + i, toOffset + i); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java new file mode 100644 index 00000000000..435d3d8ccf2 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import java.util.Collections; +import java.util.Iterator; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.AddOrGetResult; +import org.apache.arrow.vector.BaseValueVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; + +public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { + + public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; + public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + public final static String DATA_VECTOR_NAME = "$data$"; + + protected final UInt4Vector offsets; + protected FieldVector vector; + protected final CallBack callBack; + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, DEFAULT_DATA_VECTOR, callBack); + } + + protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { + super(name, allocator); + this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); + this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); + this.callBack = callBack; + } + + @Override + public boolean allocateNewSafe() { + /* boolean to keep track if all the memory allocation were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + if (!offsets.allocateNewSafe()) { + return false; + } + success = vector.allocateNewSafe(); + } finally { + if (!success) { + clear(); + } + } + offsets.zeroVector(); + return success; + } + + @Override + public void reAlloc() { + offsets.reAlloc(); + vector.reAlloc(); + } + + @Override + public UInt4Vector getOffsetVector() { + return offsets; + } + + @Override + public FieldVector getDataVector() { + return vector; + } + + @Override + public void setInitialCapacity(int numRecords) { + offsets.setInitialCapacity(numRecords + 1); + vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); + } + + @Override + public int getValueCapacity() { + final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); + if (vector == DEFAULT_DATA_VECTOR) { + return offsetValueCapacity; + } + return Math.min(vector.getValueCapacity(), offsetValueCapacity); + } + + @Override + public int getBufferSize() { + if (getAccessor().getValueCount() == 0) { + return 0; + } + return offsets.getBufferSize() + vector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + if (valueCount == 0) { + return 0; + } + + return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return Collections.singleton(getDataVector()).iterator(); + } + + @Override + public void clear() { + offsets.clear(); + vector.clear(); + super.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); + } + return buffers; + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return vector == DEFAULT_DATA_VECTOR ? 0 : 1; + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + boolean created = false; + if (vector instanceof ZeroVector) { + vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); + // returned vector must have the same field + created = true; + if (callBack != null && + // not a schema change if changing from ZeroVector to ZeroVector + (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { + callBack.doWork(); + } + } + + if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { + final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", + fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); + throw new SchemaChangeRuntimeException(msg); + } + + return new AddOrGetResult<>((T) vector, created); + } + + protected void replaceDataVector(FieldVector v) { + vector.clear(); + vector = v; + } + + public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { + + @Override + public int getValueCount() { + return Math.max(offsets.getAccessor().getValueCount() - 1, 0); + } + + @Override + public int getInnerValueCount() { + return vector.getAccessor().getValueCount(); + } + + @Override + public int getInnerValueCountAt(int index) { + return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); + } + + @Override + public boolean isNull(int index) { + return false; + } + + @Override + public boolean isEmpty(int index) { + return false; + } + } + + public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { + + @Override + public int startNewValue(int index) { + while (offsets.getValueCapacity() <= index) { + offsets.reAlloc(); + } + int offset = offsets.getAccessor().get(index); + offsets.getMutator().setSafe(index + 1, offset); + setValueCount(index + 1); + return offset; + } + + @Override + public void setValueCount(int valueCount) { + // TODO: populate offset end points + offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); + final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); + vector.getMutator().setValueCount(childValueCount); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java new file mode 100644 index 00000000000..0843907d9df --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java @@ -0,0 +1,298 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static java.util.Collections.singletonList; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.SchemaChangeRuntimeException; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { + + public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); + return new LegacyFixedSizeListVector(name, allocator, fieldType, null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + /* delegate */ + private final FixedSizeListVector fixedSizeListVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + int listSize, + DictionaryEncoding dictionary, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback); + } + + public LegacyFixedSizeListVector(String name, + BufferAllocator allocator, + FieldType fieldType, + CallBack schemaChangeCallback) { + super(name, allocator); + fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); + } + + @Override + public Field getField() { + return fixedSizeListVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.FIXED_SIZE_LIST; + } + + public int getListSize() { + return fixedSizeListVector.getListSize(); + } + + @Override + public void initializeChildrenFromFields(List children) { + fixedSizeListVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return fixedSizeListVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return fixedSizeListVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return fixedSizeListVector.getFieldInnerVectors(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionFixedSizeListReader getReader() { + return fixedSizeListVector.getReader(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + fixedSizeListVector.allocateNew(); + } + + @Override + public boolean allocateNewSafe() { + return fixedSizeListVector.allocateNewSafe(); + } + + @Override + public void reAlloc() { + fixedSizeListVector.reAlloc(); + } + + public FieldVector getDataVector() { + return fixedSizeListVector.getDataVector(); + } + + @Override + public void setInitialCapacity(int numRecords) { + fixedSizeListVector.setInitialCapacity(numRecords); + } + + @Override + public int getValueCapacity() { + return fixedSizeListVector.getValueCapacity(); + } + + @Override + public int getBufferSize() { + return fixedSizeListVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(int valueCount) { + return fixedSizeListVector.getBufferSizeFor(valueCount); + } + + @Override + public Iterator iterator() { + return fixedSizeListVector.iterator(); + } + + @Override + public void clear() { + fixedSizeListVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return fixedSizeListVector.getBuffers(clear); + } + + /** + * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 + */ + public int size() { + return fixedSizeListVector.size(); + } + + @Override + @SuppressWarnings("unchecked") + public AddOrGetResult addOrGetVector(FieldType type) { + return fixedSizeListVector.addOrGetVector(type); + } + + public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFromSafe(inIndex, outIndex, from); + } + + public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { + fixedSizeListVector.copyFrom(fromIndex, thisIndex, from); + } + + @Override + public UnionVector promoteToUnion() { + return fixedSizeListVector.promoteToUnion(); + } + + @Override + public long getValidityBufferAddress() { + return fixedSizeListVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return fixedSizeListVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return fixedSizeListVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return fixedSizeListVector.isNull(index); + } + + @Override + public int getNullCount() { + return fixedSizeListVector.getNullCount(); + } + + @Override + public int getValueCount() { + return fixedSizeListVector.getValueCount(); + } + } + + public class Mutator extends BaseValueVector.BaseMutator { + + public void setNull(int index) { + fixedSizeListVector.setNull(index); + } + + public void setNotNull(int index) { + fixedSizeListVector.setNotNull(index); + } + + @Override + public void setValueCount(int valueCount) { + fixedSizeListVector.setValueCount(valueCount); + } + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return fixedSizeListVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return fixedSizeListVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java new file mode 100644 index 00000000000..732699ada7a --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java @@ -0,0 +1,288 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Collections.list; +import static java.util.Collections.singletonList; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.ComplexCopier; +import org.apache.arrow.vector.complex.impl.UnionListReader; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Null; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector { + + public static LegacyListVector empty(String name, BufferAllocator allocator) { + return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); + } + + private Mutator mutator = new Mutator(); + private Accessor accessor = new Accessor(); + + private final ListVector listVector; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, callBack); + } + + // deprecated, use FieldType or static constructor instead + @Deprecated + public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, dictionary, callBack); + } + + public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + listVector = new ListVector(name, allocator, fieldType, callBack); + } + + @Override + public void initializeChildrenFromFields(List children) { + listVector.initializeChildrenFromFields(children); + } + + @Override + public List getChildrenFromFields() { + return listVector.getChildrenFromFields(); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + listVector.loadFieldBuffers(fieldNode, ownBuffers); + } + + @Override + public List getFieldBuffers() { + return listVector.getFieldBuffers(); + } + + @Override + public List getFieldInnerVectors() { + return listVector.getFieldInnerVectors(); + } + + public UnionListWriter getWriter() { + return listVector.getWriter(); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + listVector.allocateNew(); + } + + @Override + public void reAlloc() { + listVector.reAlloc(); + } + + public void copyFromSafe(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + public void copyFrom(int inIndex, int outIndex, ListVector from) { + listVector.copyFrom(inIndex, outIndex, from); + } + + @Override + public FieldVector getDataVector() { + return listVector.getDataVector(); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return listVector.getTransferPair(ref, allocator); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return listVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector target) { + return listVector.makeTransferPair(((LegacyListVector)target).listVector); + } + + @Override + public long getValidityBufferAddress() { + return listVector.getValidityBufferAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + return listVector.getOffsetBufferAddress(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return listVector.getValidityBuffer(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + return listVector.getOffsetBuffer(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + @Override + public UnionListReader getReader() { + return listVector.getReader(); + } + + @Override + public boolean allocateNewSafe() { + return listVector.allocateNewSafe(); + } + + public AddOrGetResult addOrGetVector(FieldType fieldType) { + return listVector.addOrGetVector(fieldType); + } + + @Override + public int getBufferSize() { + return listVector.getBufferSize(); + } + + @Override + public Field getField() { + return listVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.LIST; + } + + @Override + public void clear() { + listVector.clear(); + } + + @Override + public ArrowBuf[] getBuffers(boolean clear) { + return listVector.getBuffers(clear); + } + + @Override + public UnionVector promoteToUnion() { + return listVector.promoteToUnion(); + } + + private int lastSet = 0; + + public class Accessor extends LegacyBaseRepeatedAccessor { + + @Override + public Object getObject(int index) { + return listVector.getObject(index); + } + + @Override + public boolean isNull(int index) { + return listVector.isNull(index); + } + + @Override + public int getNullCount() { + return listVector.getNullCount(); + } + } + + public class Mutator extends LegacyBaseRepeatedMutator { + public void setNotNull(int index) { + listVector.setNotNull(index); + } + + @Override + public int startNewValue(int index) { + return listVector.startNewValue(index); + } + + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + listVector.endValue(index, size); + } + + @Override + public void setValueCount(int valueCount) { + listVector.setValueCount(valueCount); + } + + public void setLastSet(int value) { + listVector.setLastSet(value); + } + + public int getLastSet() { + return listVector.getLastSet(); + } + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java new file mode 100644 index 00000000000..77556ab774f --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; + +import io.netty.buffer.ArrowBuf; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringHashMap; +import org.apache.arrow.vector.util.TransferPair; + +public class LegacyMapVector extends AbstractMapVector { + + /* delegate */ + private final MapVector mapVector; + + public static LegacyMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + return new LegacyMapVector(name, allocator, fieldType, null); + } + + private final Accessor accessor = new Accessor(); + private final Mutator mutator = new Mutator(); + + @Deprecated + public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + } + + public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + mapVector = new MapVector(name, allocator, fieldType, callBack); + } + + @Override + public FieldReader getReader() { + return mapVector.getReader(); + } + + public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { + mapVector.copyFromSafe(fromIndex, thisIndex, from); + } + + @Override + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return mapVector.fieldNameIterator(); + } + + @Override + public void setInitialCapacity(int numRecords) { + mapVector.setInitialCapacity(numRecords); + } + + @Override + public int getBufferSize() { + return mapVector.getBufferSize(); + } + + @Override + public int getBufferSizeFor(final int valueCount) { + return (int) mapVector.getBufferSizeFor(valueCount); + } + + @Override + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator, null); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return mapVector.getTransferPair(ref, allocator, callBack); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return mapVector.getTransferPair(ref, allocator); + } + + @Override + public int getValueCapacity() { + return mapVector.getValueCapacity(); + } + + @Override + public Accessor getAccessor() { + return accessor; + } + + @Override + public Mutator getMutator() { + return mutator; + } + + public class Accessor extends BaseValueVector.BaseAccessor { + + @Override + public Object getObject(int index) { + return mapVector.getObject(index); + } + + public void get(int index, ComplexHolder holder) { + mapVector.get(index, holder); + } + + @Override + public int getValueCount() { + return mapVector.getValueCount(); + } + } + + public ValueVector getVectorById(int id) { + return mapVector.getVectorById(id); + } + + public class Mutator extends BaseValueVector.BaseMutator { + + @Override + public void setValueCount(int valueCount) { + mapVector.setValueCount(valueCount); + } + + @Override + public void reset() { } + + @Override + public void generateTestData(int values) { } + } + + @Override + public void clear() { + mapVector.clear(); + } + + @Override + public Field getField() { + return mapVector.getField(); + } + + @Override + public MinorType getMinorType() { + return MinorType.MAP; + } + + @Override + public void close() { + mapVector.close(); + } + + public void initializeChildrenFromFields(List children) { + mapVector.initializeChildrenFromFields(children); + } + + public List getChildrenFromFields() { + return mapVector.getChildren(); + } + + public boolean isNull(int index) { return false; } + + public int getNullCount() { return 0; } + + public int getValueCount() { return 0; } + + public void setValueCount(int valueCount) { } + + public Object getObject(int index) { return null; } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 4b2c9139219..ea577208cc2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -22,26 +22,22 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.singletonList; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import com.google.common.collect.ImmutableList; import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.BufferBacked; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.NullableIntVector; -import org.apache.arrow.vector.NullableVarCharVector; +import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -55,6 +51,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.JsonStringArrayList; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector { @@ -63,14 +60,12 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - final UInt4Vector offsets; - final BitVector bits; - private final List innerVectors; - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); + private ArrowBuf validityBuffer; private UnionListReader reader; private CallBack callBack; private final FieldType fieldType; + private int validityAllocationSizeInBytes; + private int lastSet; // deprecated, use FieldType or static constructor instead @Deprecated @@ -86,12 +81,12 @@ public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dic public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); - this.bits = new BitVector("$bits$", allocator); - this.offsets = getOffsetVector(); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits, offsets)); + this.validityBuffer = allocator.getEmpty(); this.reader = new UnionListReader(this); this.fieldType = checkNotNull(fieldType); this.callBack = callBack; + this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + this.lastSet = 0; } @Override @@ -115,20 +110,43 @@ public List getChildrenFromFields() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - // variable width values: truncate offset vector buffer to size (#1) - org.apache.arrow.vector.BaseDataValueVector.truncateBufferBasedOnSize(ownBuffers, 1, offsets.getBufferSizeFor(fieldNode.getLength() + 1)); - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + + validityAllocationSizeInBytes = validityBuffer.capacity(); + offsetAllocationSizeInBytes = offsetBuffer.capacity(); + lastSet = fieldNode.getLength(); + valueCount = fieldNode.getLength(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(2); + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.readerIndex(0); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + + result.add(validityBuffer); + result.add(offsetBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } public UnionListWriter getWriter() { @@ -137,14 +155,70 @@ public UnionListWriter getWriter() { @Override public void allocateNew() throws OutOfMemoryException { - super.allocateNewSafe(); - bits.allocateNewSafe(); + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } + } + + public boolean allocateNewSafe() { + boolean success = false; + try { + /* allocate validity buffer */ + allocateValidityBuffer(validityAllocationSizeInBytes); + /* allocate offset and data buffer */ + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; + } + } + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + /* reallocate the offset and data */ super.reAlloc(); - bits.reAlloc(); + } + + private void reallocValidityAndOffsetBuffers() { + reallocOffsetBuffer(); + reallocValidityBuffer(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; } public void copyFromSafe(int inIndex, int outIndex, ListVector from) { @@ -181,7 +255,7 @@ public TransferPair makeTransferPair(ValueVector target) { @Override public long getValidityBufferAddress() { - return (bits.getDataBuffer().memoryAddress()); + return (validityBuffer.memoryAddress()); } @Override @@ -191,11 +265,11 @@ public long getDataBufferAddress() { @Override public long getOffsetBufferAddress() { - return (offsets.getDataBuffer().memoryAddress()); + return (offsetBuffer.memoryAddress()); } @Override - public ArrowBuf getValidityBuffer() { return bits.getDataBuffer(); } + public ArrowBuf getValidityBuffer() { return validityBuffer; } @Override public ArrowBuf getDataBuffer() { @@ -203,17 +277,13 @@ public ArrowBuf getDataBuffer() { } @Override - public ArrowBuf getOffsetBuffer() { return offsets.getDataBuffer(); } + public ArrowBuf getOffsetBuffer() { return offsetBuffer; } private class TransferImpl implements TransferPair { ListVector to; - TransferPair bitsTransferPair; - TransferPair offsetsTransferPair; TransferPair dataTransferPair; - TransferPair[] pairs; - public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { this(new ListVector(name, allocator, fieldType, callBack)); } @@ -221,39 +291,99 @@ public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { public TransferImpl(ListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); - offsetsTransferPair = offsets.makeTransferPair(to.offsets); - bitsTransferPair = bits.makeTransferPair(to.bits); if (to.getDataVector() instanceof ZeroVector) { to.addOrGetVector(vector.getField().getFieldType()); } dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); - pairs = new TransferPair[] {bitsTransferPair, offsetsTransferPair, dataTransferPair}; } @Override public void transfer() { - for (TransferPair pair : pairs) { - pair.transfer(); - } + dataTransferPair.transfer(); + to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; + to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; + to.setValueCount(valueCount); } @Override public void splitAndTransfer(int startIndex, int length) { - UInt4Vector.Accessor offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final int startPoint = offsetVectorAccessor.get(startIndex); - final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; + final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int sliceLength = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH) - startPoint; to.clear(); - to.offsets.allocateNew(length + 1); - offsetVectorAccessor = ListVector.this.offsets.getAccessor(); - final UInt4Vector.Mutator targetOffsetVectorMutator = to.offsets.getMutator(); + to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + /* splitAndTransfer offset buffer */ for (int i = 0; i < length + 1; i++) { - targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint); + final int relativeOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - startPoint; + to.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeOffset); } - bitsTransferPair.splitAndTransfer(startIndex, length); + /* splitAndTransfer validity buffer */ + splitAndTransferValidityBuffer(startIndex, length, to); + /* splitAndTransfer data buffer */ dataTransferPair.splitAndTransfer(startPoint, sliceLength); to.lastSet = length; - to.mutator.setValueCount(length); + to.setValueCount(length); + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } } @Override @@ -267,47 +397,11 @@ public void copyValueSafe(int from, int to) { } } - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - @Override public UnionListReader getReader() { return reader; } - @Override - public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - success = success && bits.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - if (success) { - offsets.zeroVector(); - bits.zeroVector(); - } - return success; - } - public AddOrGetResult addOrGetVector(FieldType fieldType) { AddOrGetResult result = super.addOrGetVector(fieldType); reader = new UnionListReader(this); @@ -316,10 +410,12 @@ public AddOrGetResult addOrGetVector(FieldType fieldT @Override public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { + if (getValueCount() == 0) { return 0; } - return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize(); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + return offsetBufferSize + validityBufferSize + vector.getBufferSize(); } @Override @@ -334,16 +430,14 @@ public MinorType getMinorType() { @Override public void clear() { - offsets.clear(); - vector.clear(); - bits.clear(); - lastSet = 0; super.clear(); + validityBuffer = releaseBuffer(validityBuffer); + lastSet = 0; } @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false), + final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); if (clear) { for (ArrowBuf buffer : buffers) { @@ -365,99 +459,97 @@ public UnionVector promoteToUnion() { return vector; } - private int lastSet = 0; + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final List vals = new JsonStringArrayList<>(); + final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + final ValueVector vv = getDataVector(); + for (int i = start; i < end; i++) { + vals.add(vv.getObject(i)); + } - public class Accessor extends BaseRepeatedAccessor { + return vals; + } - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } - final List vals = new JsonStringArrayList<>(); - final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor(); - final int start = offsetsAccessor.get(index); - final int end = offsetsAccessor.get(index + 1); - final ValueVector vv = getDataVector(); - if (vv instanceof NullableIntVector || vv instanceof NullableVarCharVector) { - for (int i = start; i < end; i++) { - vals.add(vv.getObject(i)); - } - } else { - final ValueVector.Accessor valuesAccessor = vv.getAccessor(); - for (int i = start; i < end; i++) { - vals.add(valuesAccessor.getObject(i)); - } - } + @Override + public boolean isNull(int index) { + return (isSet(index) == 0); + } - return vals; - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public boolean isNull(int index) { - return bits.getAccessor().get(index) == 0; - } + @Override + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); + public void setNotNull(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + lastSet = index + 1; } - public class Mutator extends BaseRepeatedMutator { - public void setNotNull(int index) { - bits.getMutator().setSafe(index, 1); - lastSet = index + 1; + @Override + public int startNewValue(int index) { + if (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); } - - @Override - public int startNewValue(int index) { - for (int i = lastSet; i <= index; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - setNotNull(index); - lastSet = index + 1; - return offsets.getAccessor().get(lastSet); + for (int i = lastSet; i <= index; i++) { + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } + setNotNull(index); + lastSet = index + 1; + return offsetBuffer.getInt(lastSet * OFFSET_WIDTH); + } - /** - * End the current value - * - * @param index index of the value to end - * @param size number of elements in the list that was written - */ - public void endValue(int index, int size) { - offsets.getMutator().set(index + 1, offsets.getAccessor().get(index + 1) + size); - } + /** + * End the current value + * + * @param index index of the value to end + * @param size number of elements in the list that was written + */ + public void endValue(int index, int size) { + final int currentOffset = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); + } - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - if (valueCount == 0) { - offsets.getMutator().setValueCount(0); - } else { - for (int i = lastSet; i < valueCount; i++) { - offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i)); - } - offsets.getMutator().setValueCount(valueCount + 1); + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + if (valueCount > 0) { + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityAndOffsetBuffers(); } - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - if (vector instanceof NullableIntVector || vector instanceof NullableVarCharVector) { - vector.setValueCount(childValueCount); - } else { - vector.getMutator().setValueCount(childValueCount); + for (int i = lastSet; i < valueCount; i++) { + /* fill the holes with offsets */ + final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); + offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } - - bits.getMutator().setValueCount(valueCount); - } - - public void setLastSet(int value) { - lastSet = value; } + /* valueCount for the data vector is the current end offset */ + final int childValueCount = (valueCount == 0) ? 0 : + offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + vector.setValueCount(childValueCount); + } - public int getLastSet() { - return lastSet; - } + public void setLastSet(int value) { + lastSet = value; } + public int getLastSet() { + return lastSet; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 95efa608718..6089a67924f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -56,8 +56,6 @@ public static MapVector empty(String name, BufferAllocator allocator) { } private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); protected final FieldType fieldType; public int valueCount; @@ -70,6 +68,7 @@ public MapVector(String name, BufferAllocator allocator, CallBack callBack) { public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; } @Override @@ -230,7 +229,7 @@ public void splitAndTransfer(int startIndex, int length) { for (TransferPair p : pairs) { p.splitAndTransfer(startIndex, length); } - to.getMutator().setValueCount(length); + to.setValueCount(length); } } @@ -255,76 +254,54 @@ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { @Override public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("accessor is not needed for MAP"); } @Override public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("mutator is not needed for MAP"); } - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v instanceof NullableVarCharVector || v instanceof NullableIntVector) { - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } - } - } else { - if (v != null && index < v.getAccessor().getValueCount()) { - Object value = v.getAccessor().getObject(index); - if (value != null) { - vv.put(child, value); - } - } + @Override + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); } } - return vv; } + return vv; + } - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; - } + @Override + public boolean isNull(int index) { return false; } + @Override + public int getNullCount() { return 0; } - @Override - public int getValueCount() { - return valueCount; - } + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; } - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); + @Override + public int getValueCount() { + return valueCount; } - public class Mutator extends BaseValueVector.BaseMutator { - - @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - if (v instanceof NullableIntVector || v instanceof NullableVarCharVector) { - v.setValueCount(valueCount); - } else { - v.getMutator().setValueCount(valueCount); - } - } - MapVector.this.valueCount = valueCount; - } - - @Override - public void reset() { - } + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} - @Override - public void generateTestData(int values) { + @Override + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); } + MapVector.this.valueCount = valueCount; } @Override @@ -372,11 +349,4 @@ public void initializeChildrenFromFields(List children) { public List getChildrenFromFields() { return getChildren(); } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 1c9d5aa2659..a1478c5a721 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -27,13 +28,9 @@ import com.google.common.collect.ObjectArrays; import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BaseDataValueVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.NullableVectorDefinitionSetter; -import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; import org.apache.arrow.vector.holders.ComplexHolder; @@ -44,6 +41,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; public class NullableMapVector extends MapVector implements FieldVector { @@ -56,12 +54,8 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) { private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); - protected final BitVector bits; - - private final List innerVectors; - - private final Accessor accessor; - private final Mutator mutator; + private ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @Deprecated @@ -77,10 +71,8 @@ public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncod public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, checkNotNull(allocator), fieldType, callBack); - this.bits = new BitVector("$bits$", allocator); - this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits)); - this.accessor = new Accessor(); - this.mutator = new Mutator(); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); } @Override @@ -92,18 +84,33 @@ public Field getField() { @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - BaseDataValueVector.load(fieldNode, getFieldInnerVectors(), ownBuffers); - this.valueCount = fieldNode.getLength(); + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = bitBuffer.retain(allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); } @Override public List getFieldBuffers() { - return BaseDataValueVector.unload(getFieldInnerVectors()); + List result = new ArrayList<>(1); + + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + result.add(validityBuffer); + + return result; } @Override + @Deprecated public List getFieldInnerVectors() { - return innerVectors; + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); } @Override @@ -146,49 +153,129 @@ protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, @Override public void transfer() { - bits.transferTo(target.bits); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; super.transfer(); } @Override public void copyValueSafe(int fromIndex, int toIndex) { - target.bits.copyFromSafe(fromIndex, toIndex, bits); + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); super.copyValueSafe(fromIndex, toIndex); } @Override public void splitAndTransfer(int startIndex, int length) { - bits.splitAndTransferTo(startIndex, length, target.bits); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); super.splitAndTransfer(startIndex, length); } } + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } + else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } + else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + @Override public int getValueCapacity() { - return Math.min(bits.getValueCapacity(), super.getValueCapacity()); + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); } @Override public ArrowBuf[] getBuffers(boolean clear) { - return ObjectArrays.concat(bits.getBuffers(clear), super.getBuffers(clear), ArrowBuf.class); + if (clear) { + validityBuffer.retain(1); + } + return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class); } @Override public void close() { - bits.close(); + clearValidityBuffer(); super.close(); } @Override public void clear() { - bits.clear(); + clearValidityBuffer(); super.clear(); } + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); + } @Override public int getBufferSize() { - return super.getBufferSize() + bits.getBufferSize(); + if (valueCount == 0) { return 0; } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override @@ -197,12 +284,12 @@ public int getBufferSizeFor(final int valueCount) { return 0; } return super.getBufferSizeFor(valueCount) - + bits.getBufferSizeFor(valueCount); + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override public void setInitialCapacity(int numRecords) { - bits.setInitialCapacity(numRecords); + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); super.setInitialCapacity(numRecords); } @@ -215,25 +302,59 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - success = super.allocateNewSafe() && bits.allocateNewSafe(); + clearValidityBuffer(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); } finally { if (!success) { clear(); + return false; } } - bits.zeroVector(); - return success; + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int)size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); } @Override public void reAlloc() { - bits.reAlloc(); + /* reallocate the validity buffer */ + reallocValidityBuffer(); super.reAlloc(); } + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long)currentBufferCapacity) { + baseSize = (long)currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int)newAllocationSize; + } + @Override public long getValidityBufferAddress() { - return bits.getBuffer().memoryAddress(); + return validityBuffer.memoryAddress(); } @Override @@ -248,7 +369,7 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getValidityBuffer() { - return bits.getDataBuffer(); + return validityBuffer; } @Override @@ -261,82 +382,76 @@ public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public final class Accessor extends MapVector.Accessor { - final BitVector.Accessor bAccessor = bits.getAccessor(); - - @Override - public Object getObject(int index) { - if (isNull(index)) { - return null; - } else { - return super.getObject(index); - } - } - - @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); - } - - @Override - public int getNullCount() { - return bits.getAccessor().getNullCount(); - } - - @Override - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - return bAccessor.get(index); + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); } - } - public final class Mutator extends MapVector.Mutator implements NullableVectorDefinitionSetter { + @Override + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } - private Mutator() { - } + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } - @Override - public void setIndexDefined(int index) { - bits.getMutator().setSafe(index, 1); - } + public boolean isNull(int index) { + return isSet(index) == 0; + } - public void setNull(int index) { - bits.getMutator().setSafe(index, 0); - } + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - super.setValueCount(valueCount); - bits.getMutator().setValueCount(valueCount); + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } - @Override - public void generateTestData(int valueCount) { - super.generateTestData(valueCount); - bits.getMutator().generateTestDataAlt(valueCount); + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } - @Override - public void reset() { - bits.getMutator().setValueCount(0); + @Override + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); } + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + public void reset() { + valueCount = 0; } @Override + @Deprecated public Accessor getAccessor() { - return accessor; + throw new UnsupportedOperationException("Accessor is not supported for reading from Nullable MAP"); } @Override + @Deprecated public Mutator getMutator() { - return mutator; + throw new UnsupportedOperationException("Mutator is not supported for writing to Nullable MAP"); } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index 614c266acf1..06b0f4d4327 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -52,6 +52,6 @@ public void copyAsField(String name, MapWriter writer) { @Override public boolean isSet() { - return !nullableMapVector.getAccessor().isNull(idx()); + return !nullableMapVector.isNull(idx()); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 9722196ed7c..5bd439cac42 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -162,7 +162,7 @@ private FieldWriter promoteToUnion() { writer = new UnionWriter(unionVector, nullableMapWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { - unionVector.getMutator().setType(i, vector.getMinorType()); + unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index 3ebd0cd7dd9..c77ca4e8f23 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -78,7 +78,7 @@ public void setPosition(int index) { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java index f3e9b8773f2..4ad2f6f5fad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java @@ -46,7 +46,7 @@ public UnionFixedSizeListReader(FixedSizeListVector vector) { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } @Override @@ -56,7 +56,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java index b98c36d2bf7..6243a2833c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java @@ -19,6 +19,7 @@ package org.apache.arrow.vector.complex.impl; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.vector.UInt4Vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; @@ -33,12 +34,11 @@ public class UnionListReader extends AbstractFieldReader { private ListVector vector; private ValueVector data; - private UInt4Vector offsets; + private static final int OFFSET_WIDTH = 4; public UnionListReader(ListVector vector) { this.vector = vector; this.data = vector.getDataVector(); - this.offsets = vector.getOffsetVector(); } @Override @@ -48,7 +48,7 @@ public Field getField() { @Override public boolean isSet() { - return !vector.getAccessor().isNull(idx()); + return !vector.isNull(idx()); } private int currentOffset; @@ -57,8 +57,8 @@ public boolean isSet() { @Override public void setPosition(int index) { super.setPosition(index); - currentOffset = offsets.getAccessor().get(index) - 1; - maxOffset = offsets.getAccessor().get(index + 1); + currentOffset = vector.getOffsetBuffer().getInt(index * OFFSET_WIDTH) - 1; + maxOffset = vector.getOffsetBuffer().getInt((index + 1) * OFFSET_WIDTH); } @Override @@ -68,7 +68,7 @@ public FieldReader reader() { @Override public Object readObject() { - return vector.getAccessor().getObject(idx()); + return vector.getObject(idx()); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 3b7dc4a56e1..762a442c983 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -47,11 +47,10 @@ public class DictionaryEncoder { public static ValueVector encode(ValueVector vector, Dictionary dictionary) { validateType(vector.getMinorType()); // load dictionary values into a hashmap for lookup - ValueVector.Accessor dictionaryAccessor = dictionary.getVector().getAccessor(); - Map lookUps = new HashMap<>(dictionaryAccessor.getValueCount()); - for (int i = 0; i < dictionaryAccessor.getValueCount(); i++) { + Map lookUps = new HashMap<>(dictionary.getVector().getValueCount()); + for (int i = 0; i < dictionary.getVector().getValueCount(); i++) { // for primitive array types we need a wrapper that implements equals and hashcode appropriately - lookUps.put(dictionaryAccessor.getObject(i), i); + lookUps.put(dictionary.getVector().getObject(i), i); } Field valueField = vector.getField(); @@ -61,14 +60,13 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { // vector to hold our indices (dictionary encoded values) FieldVector indices = indexField.createVector(vector.getAllocator()); - ValueVector.Mutator mutator = indices.getMutator(); // use reflection to pull out the set method // TODO implement a common interface for int vectors Method setter = null; for (Class c : ImmutableList.of(int.class, long.class)) { try { - setter = mutator.getClass().getMethod("setSafe", int.class, c); + setter = indices.getClass().getMethod("setSafe", int.class, c); break; } catch (NoSuchMethodException e) { // ignore @@ -78,21 +76,20 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new IllegalArgumentException("Dictionary encoding does not have a valid int type:" + indices.getClass()); } - ValueVector.Accessor accessor = vector.getAccessor(); - int count = accessor.getValueCount(); + int count = vector.getValueCount(); indices.allocateNew(); try { for (int i = 0; i < count; i++) { - Object value = accessor.getObject(i); + Object value = vector.getObject(i); if (value != null) { // if it's null leave it null // note: this may fail if value was not included in the dictionary Object encoded = lookUps.get(value); if (encoded == null) { throw new IllegalArgumentException("Dictionary encoding not defined for value:" + value); } - setter.invoke(mutator, i, encoded); + setter.invoke(indices, i, encoded); } } } catch (IllegalAccessException e) { @@ -101,7 +98,7 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { throw new RuntimeException("InvocationTargetException invoking vector mutator set():", e.getCause()); } - mutator.setValueCount(count); + indices.setValueCount(count); return indices; } @@ -114,15 +111,14 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { - ValueVector.Accessor accessor = indices.getAccessor(); - int count = accessor.getValueCount(); + int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); - int dictionaryCount = dictionaryVector.getAccessor().getValueCount(); + int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { - Object index = accessor.getObject(i); + Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { @@ -133,7 +129,7 @@ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); - decoded.getMutator().setValueCount(count); + decoded.setValueCount(count); return decoded; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java index b35aba5426e..7dc10b5e629 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java @@ -82,7 +82,7 @@ protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, Writab for (long id : dictionaryIdsUsed) { Dictionary dictionary = provider.lookup(id); FieldVector vector = dictionary.getVector(); - int count = vector.getAccessor().getValueCount(); + int count = vector.getValueCount(); VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), count); VectorUnloader unloader = new VectorUnloader(dictRoot); ArrowRecordBatch batch = unloader.getRecordBatch(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 0de804406f7..b21fcaac391 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -23,60 +23,28 @@ import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.schema.ArrowVectorType.OFFSET; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; import java.io.File; import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.*; import com.google.common.collect.ImmutableList; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.NullableVarBinaryVector; -import org.apache.arrow.vector.NullableVarCharVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroTZVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliTZVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoTZVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecTZVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.UInt1Vector; -import org.apache.arrow.vector.UInt2Vector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.UInt8Vector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowVectorType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.DictionaryUtility; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -153,7 +121,7 @@ private void readDictionaryBatches() throws JsonParseException, IOException { FieldVector vector = dict.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); read(root); readToken(END_OBJECT); @@ -177,7 +145,7 @@ public boolean read(VectorSchemaRoot root) throws IOException { { for (Field field : root.getSchema().getFields()) { FieldVector vector = root.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -204,7 +172,7 @@ public VectorSchemaRoot read() throws IOException { { for (Field field : schema.getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - readVector(field, vector); + readFromJsonIntoVector(field, vector); } } readToken(END_ARRAY); @@ -218,16 +186,24 @@ public VectorSchemaRoot read() throws IOException { } } - /** - * TODO: A better way of implementing this function is to use `loadFieldBuffers` methods in - * FieldVector to set the inner-vector data as done in `ArrowFileReader`. - */ - private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException { + private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); - } + ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; + /* + * The order of inner buffers is : + * Fixed width vector: + * -- validity buffer + * -- data buffer + * Variable width vector: + * -- validity buffer + * -- offset buffer + * -- data buffer + * + * This is similar to what getFieldInnerVectors() used to give but now that we don't have + * inner vectors anymore, we will work directly at the buffer level -- populate buffers + * locally as we read from Json parser and do loadFieldBuffers on the vector followed by + * releasing the local buffers. + */ readToken(START_OBJECT); { // If currently reading dictionaries, field name is not important so don't check @@ -236,39 +212,41 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name); } - // Initialize the vector with required capacity - int count = readNextField("count", Integer.class); - vector.setInitialCapacity(count); - vector.allocateNew(); + /* Initialize the vector with required capacity but don't allocate since we would + * be doing loadFieldBuffers. + */ + int valueCount = readNextField("count", Integer.class); + vector.setInitialCapacity(valueCount); - // Read inner vectors for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - ValueVector valueVector = (ValueVector) fieldInnerVectors.get(v); nextFieldIs(vectorType.getName()); readToken(START_ARRAY); - int innerVectorCount = vectorType.equals(OFFSET) ? count + 1 : count; - for (int i = 0; i < innerVectorCount; i++) { + int innerBufferValueCount = valueCount; + if (vectorType.equals(OFFSET)) { + /* offset buffer has 1 additional value capacity */ + innerBufferValueCount = valueCount + 1; + } + for (int i = 0; i < innerBufferValueCount; i++) { + /* write data to the buffer */ parser.nextToken(); - setValueFromParser(valueVector, i); + /* for variable width vectors, value count doesn't help pre-determining the capacity of + * the underlying data buffer. So we need to pass down the offset buffer (which was already + * populated in the previous iteration of this loop). + */ + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR + || vector.getMinorType() == Types.MinorType.VARBINARY)) { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + vectorBuffers[v-1], i, innerBufferValueCount); + } else { + vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], + null, i, innerBufferValueCount); + } } readToken(END_ARRAY); } - // Set lastSet before valueCount to prevent setValueCount from filling empty values - switch (vector.getMinorType()) { - case LIST: - // ListVector starts lastSet from index 0, so lastSet value is always last index written + 1 - ((ListVector) vector).getMutator().setLastSet(count); - break; - case VARBINARY: - ((NullableVarBinaryVector) vector).getMutator().setLastSet(count - 1); - break; - case VARCHAR: - ((NullableVarCharVector) vector).setLastSet(count - 1); - break; - } - vector.getMutator().setValueCount(count); + vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); // read child vectors, if any List fields = field.getChildren(); @@ -282,12 +260,16 @@ private void readVector(Field field, FieldVector vector) throws JsonParseExcepti for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = vectorChildren.get(i); - readVector(childField, childVector); + readFromJsonIntoVector(childField, childVector); } readToken(END_ARRAY); } } readToken(END_OBJECT); + + for (ArrowBuf buffer: vectorBuffers) { + buffer.release(); + } } private byte[] decodeHexSafe(String hexString) throws IOException { @@ -298,99 +280,124 @@ private byte[] decodeHexSafe(String hexString) throws IOException { } } - private void setValueFromParser(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case BIT: - ((BitVector) valueVector).getMutator().set(i, parser.readValueAs(Boolean.class) ? 1 : 0); - break; - case TINYINT: - ((TinyIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case SMALLINT: - ((SmallIntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case INT: - ((IntVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case BIGINT: - ((BigIntVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case UINT1: - ((UInt1Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT2: - ((UInt2Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT4: - ((UInt4Vector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case UINT8: - ((UInt8Vector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case FLOAT4: - ((Float4Vector) valueVector).getMutator().set(i, parser.readValueAs(Float.class)); - break; - case FLOAT8: - ((Float8Vector) valueVector).getMutator().set(i, parser.readValueAs(Double.class)); - break; - case DECIMAL: { - DecimalVector decimalVector = (DecimalVector) valueVector; - // Here we assume the decimal value is the unscaled integer value as a string - BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); - DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, decimalVector.getBuffer(), i); - } - break; - case VARBINARY: - ((VarBinaryVector) valueVector).getMutator().setSafe(i, decodeHexSafe(parser.readValueAs(String.class))); - break; - case VARCHAR: - ((VarCharVector) valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8)); - break; - case DATEDAY: - ((DateDayVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case DATEMILLI: - ((DateMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESEC: - ((TimeSecVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMILLI: - ((TimeMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Integer.class)); - break; - case TIMEMICRO: - ((TimeMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMENANO: - ((TimeNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSEC: - ((TimeStampSecVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLI: - ((TimeStampMilliVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICRO: - ((TimeStampMicroVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANO: - ((TimeStampNanoVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSECTZ: - ((TimeStampSecTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLITZ: - ((TimeStampMilliTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICROTZ: - ((TimeStampMicroTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANOTZ: - ((TimeStampNanoTZVector) valueVector).getMutator().set(i, parser.readValueAs(Long.class)); - break; - default: - throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); + private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector, + ArrowBuf buffer, ArrowBuf offsetBuffer, int index, + int valueCount) throws IOException { + if (bufferType.equals(TYPE)) { + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + } else if (bufferType.equals(OFFSET)) { + buffer = BaseNullableVariableWidthVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + } else if (bufferType.equals(VALIDITY)) { + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case BIT: + buffer = BitVectorHelper.setValidityBit(buffer, allocator, + valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); + break; + case TINYINT: + buffer = NullableTinyIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Byte.class)); + break; + case SMALLINT: + buffer = NullableSmallIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Short.class)); + break; + case INT: + buffer = NullableIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case BIGINT: + buffer = NullableBigIntVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case FLOAT4: + buffer = NullableFloat4Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Float.class)); + break; + case FLOAT8: + buffer = NullableFloat8Vector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Double.class)); + break; + case DECIMAL: + buffer = NullableDecimalVector.set(buffer, allocator, + valueCount, index, decodeHexSafe(parser.readValueAs(String.class))); + break; + case VARBINARY: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + decodeHexSafe(parser.readValueAs(String.class)), valueCount); + break; + case VARCHAR: + assert (offsetBuffer != null); + buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, + parser.readValueAs(String.class).getBytes(UTF_8), valueCount); + break; + case DATEDAY: + buffer = NullableDateDayVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case DATEMILLI: + buffer = NullableDateMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESEC: + buffer = NullableTimeSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMILLI: + buffer = NullableTimeMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Integer.class)); + break; + case TIMEMICRO: + buffer = NullableTimeMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMENANO: + buffer = NullableTimeNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSEC: + buffer = NullableTimeStampSecVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLI: + buffer = NullableTimeStampMilliVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICRO: + buffer = NullableTimeStampMicroVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANO: + buffer = NullableTimeStampNanoVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPSECTZ: + buffer = NullableTimeStampSecTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLITZ: + buffer = NullableTimeStampMilliTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICROTZ: + buffer = NullableTimeStampMicroTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANOTZ: + buffer = NullableTimeStampNanoTZVector.set(buffer, allocator, + valueCount, index, parser.readValueAs(Long.class)); + break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } + + return buffer; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 05341bec44e..7921cd34c77 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -18,9 +18,10 @@ package org.apache.arrow.vector.file.json; +import static org.apache.arrow.vector.schema.ArrowVectorType.*; + import java.io.File; import java.io.IOException; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -28,28 +29,11 @@ import com.google.common.collect.ImmutableList; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.BufferBacked; -import org.apache.arrow.vector.DateDayVector; -import org.apache.arrow.vector.DateMilliVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TimeMicroVector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeNanoVector; -import org.apache.arrow.vector.TimeSecVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.TimeStampMilliVector; -import org.apache.arrow.vector.TimeStampNanoVector; -import org.apache.arrow.vector.TimeStampSecVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ValueVector.Accessor; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.schema.ArrowVectorType; -import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; @@ -135,7 +119,7 @@ private void writeDictionaryBatches(JsonGenerator generator, Set dictionar FieldVector vector = dictionary.getVector(); List fields = ImmutableList.of(vector.getField()); List vectors = ImmutableList.of(vector); - VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getAccessor().getValueCount()); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount()); writeBatch(root); generator.writeEndObject(); @@ -157,31 +141,36 @@ private void writeBatch(VectorSchemaRoot recordBatch) throws IOException { generator.writeArrayFieldStart("columns"); for (Field field : recordBatch.getSchema().getFields()) { FieldVector vector = recordBatch.getVector(field.getName()); - writeVector(field, vector); + writeFromVectorIntoJson(field, vector); } generator.writeEndArray(); } generator.writeEndObject(); } - private void writeVector(Field field, FieldVector vector) throws IOException { + private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); - List fieldInnerVectors = vector.getFieldInnerVectors(); - if (vectorTypes.size() != fieldInnerVectors.size()) { - throw new IllegalArgumentException("vector types and inner vectors are not the same size: " + vectorTypes.size() + " != " + fieldInnerVectors.size()); + List vectorBuffers = vector.getFieldBuffers(); + if (vectorTypes.size() != vectorBuffers.size()) { + throw new IllegalArgumentException("vector types and inner vector buffers are not the same size: " + vectorTypes.size() + " != " + vectorBuffers.size()); } generator.writeStartObject(); { generator.writeObjectField("name", field.getName()); - int valueCount = vector.getAccessor().getValueCount(); + int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); - BufferBacked innerVector = fieldInnerVectors.get(v); + ArrowBuf vectorBuffer = vectorBuffers.get(v); generator.writeArrayFieldStart(vectorType.getName()); - ValueVector valueVector = (ValueVector) innerVector; - for (int i = 0; i < valueVector.getAccessor().getValueCount(); i++) { - writeValueToGenerator(valueVector, i); + final int bufferValueCount = (vectorType.equals(OFFSET)) ? valueCount + 1 : valueCount; + for (int i = 0; i < bufferValueCount; i++) { + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || + vector.getMinorType() == Types.MinorType.VARBINARY)) { + writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i); + } else { + writeValueToGenerator(vectorType, vectorBuffer, null, vector, i); + } } generator.writeEndArray(); } @@ -195,7 +184,7 @@ private void writeVector(Field field, FieldVector vector) throws IOException { for (int i = 0; i < fields.size(); i++) { Field childField = fields.get(i); FieldVector childVector = children.get(i); - writeVector(childField, childVector); + writeFromVectorIntoJson(childField, childVector); } generator.writeEndArray(); } @@ -203,62 +192,101 @@ private void writeVector(Field field, FieldVector vector) throws IOException { generator.writeEndObject(); } - private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException { - switch (valueVector.getMinorType()) { - case DATEDAY: - generator.writeNumber(((DateDayVector) valueVector).getAccessor().get(i)); - break; - case DATEMILLI: - generator.writeNumber(((DateMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESEC: - generator.writeNumber(((TimeSecVector) valueVector).getAccessor().get(i)); - break; - case TIMEMILLI: - generator.writeNumber(((TimeMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMEMICRO: - generator.writeNumber(((TimeMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMENANO: - generator.writeNumber(((TimeNanoVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPSEC: - generator.writeNumber(((TimeStampSecVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMILLI: - generator.writeNumber(((TimeStampMilliVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPMICRO: - generator.writeNumber(((TimeStampMicroVector) valueVector).getAccessor().get(i)); - break; - case TIMESTAMPNANO: - generator.writeNumber(((TimeStampNanoVector) valueVector).getAccessor().get(i)); - break; - case BIT: - generator.writeNumber(((BitVector) valueVector).getAccessor().get(i)); - break; - case VARBINARY: { - String hexString = Hex.encodeHexString(((VarBinaryVector) valueVector).getAccessor().get(i)); - generator.writeString(hexString); + private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, + ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException { + if (bufferType.equals(TYPE)) { + generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); + } else if (bufferType.equals(OFFSET)) { + generator.writeNumber(buffer.getInt(index * BaseNullableVariableWidthVector.OFFSET_WIDTH)); + } else if(bufferType.equals(VALIDITY)) { + generator.writeNumber(vector.isNull(index) ? 0 : 1); + } else if (bufferType.equals(DATA)) { + switch (vector.getMinorType()) { + case TINYINT: + generator.writeNumber(NullableTinyIntVector.get(buffer, index)); + break; + case SMALLINT: + generator.writeNumber(NullableSmallIntVector.get(buffer, index)); + break; + case INT: + generator.writeNumber(NullableIntVector.get(buffer, index)); + break; + case BIGINT: + generator.writeNumber(NullableBigIntVector.get(buffer, index)); + break; + case FLOAT4: + generator.writeNumber(NullableFloat4Vector.get(buffer, index)); + break; + case FLOAT8: + generator.writeNumber(NullableFloat8Vector.get(buffer, index)); + break; + case DATEDAY: + generator.writeNumber(NullableDateDayVector.get(buffer, index)); + break; + case DATEMILLI: + generator.writeNumber(NullableDateMilliVector.get(buffer, index)); + break; + case TIMESEC: + generator.writeNumber(NullableTimeSecVector.get(buffer, index)); + break; + case TIMEMILLI: + generator.writeNumber(NullableTimeMilliVector.get(buffer, index)); + break; + case TIMEMICRO: + generator.writeNumber(NullableTimeMicroVector.get(buffer, index)); + break; + case TIMENANO: + generator.writeNumber(NullableTimeNanoVector.get(buffer, index)); + break; + case TIMESTAMPSEC: + generator.writeNumber(NullableTimeStampSecVector.get(buffer, index)); + break; + case TIMESTAMPMILLI: + generator.writeNumber(NullableTimeStampMilliVector.get(buffer, index)); + break; + case TIMESTAMPMICRO: + generator.writeNumber(NullableTimeStampMicroVector.get(buffer, index)); + break; + case TIMESTAMPNANO: + generator.writeNumber(NullableTimeStampNanoVector.get(buffer, index)); + break; + case TIMESTAMPSECTZ: + generator.writeNumber(NullableTimeStampSecTZVector.get(buffer, index)); + break; + case TIMESTAMPMILLITZ: + generator.writeNumber(NullableTimeStampMilliTZVector.get(buffer, index)); + break; + case TIMESTAMPMICROTZ: + generator.writeNumber(NullableTimeStampMicroTZVector.get(buffer, index)); + break; + case TIMESTAMPNANOTZ: + generator.writeNumber(NullableTimeStampNanoTZVector.get(buffer, index)); + break; + case BIT: + generator.writeNumber(BitVectorHelper.get(buffer, index)); + break; + case VARBINARY: { + assert offsetBuffer != null; + String hexString = Hex.encodeHexString(BaseNullableVariableWidthVector.get(buffer, + offsetBuffer, index)); + generator.writeObject(hexString); + break; } - break; - case DECIMAL: { - BigDecimal decimalValue = ((DecimalVector) valueVector).getAccessor().getObject(i); - // We write the unscaled value, because the scale is stored in the type metadata. - generator.writeString(decimalValue.unscaledValue().toString()); + case VARCHAR: { + assert offsetBuffer != null; + byte[] b = (BaseNullableVariableWidthVector.get(buffer, offsetBuffer, index)); + generator.writeString(new String(b, "UTF-8")); + break; } - break; - default: - // TODO: each type - Accessor accessor = valueVector.getAccessor(); - Object value = accessor.getObject(i); - if (value instanceof Number || value instanceof Boolean) { - generator.writeObject(value); - } else { - generator.writeObject(value.toString()); + case DECIMAL: { + String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer, + index)); + generator.writeString(hexString); + break; } - break; + default: + throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); + } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java index 5851bd5fa5d..c27e5e5c85c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java @@ -114,13 +114,13 @@ public static void compareFieldVectors(FieldVector vector1, FieldVector vector2) if (!field1.equals(vector2.getField())) { throw new IllegalArgumentException("Different Fields:\n" + field1 + "\n!=\n" + vector2.getField()); } - int valueCount = vector1.getAccessor().getValueCount(); - if (valueCount != vector2.getAccessor().getValueCount()) { - throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getAccessor().getValueCount()); + int valueCount = vector1.getValueCount(); + if (valueCount != vector2.getValueCount()) { + throw new IllegalArgumentException("Different value count for field " + field1 + " : " + valueCount + " != " + vector2.getValueCount()); } for (int j = 0; j < valueCount; j++) { - Object obj1 = vector1.getAccessor().getObject(j); - Object obj2 = vector2.getAccessor().getObject(j); + Object obj1 = vector1.getObject(j); + Object obj2 = vector2.getObject(j); if (!equals(field1.getType(), obj1, obj2)) { throw new IllegalArgumentException( "Different values in column:\n" + field1 + " at index " + j + ": " + obj1 + " != " + obj2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index 17fcf05fcd9..ada34149787 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -340,59 +340,56 @@ public void testReallocAfterVectorTransfer2() { int valueCapacity = vector.getValueCapacity(); assertEquals(4096, valueCapacity); - final NullableBitVector.Mutator mutator = vector.getMutator(); - final NullableBitVector.Accessor accessor = vector.getAccessor(); - for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger first realloc */ - mutator.setSafe(valueCapacity, 1, 1); + vector.setSafe(valueCapacity, 1, 1); assertEquals(valueCapacity * 2, vector.getValueCapacity()); for (int i = valueCapacity; i < valueCapacity*2; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*2; i++) { if (((i & 1) == 1) || (i == valueCapacity)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } /* trigger second realloc */ - mutator.setSafe(valueCapacity*2, 1, 1); + vector.setSafe(valueCapacity*2, 1, 1); assertEquals(valueCapacity * 4, vector.getValueCapacity()); for (int i = valueCapacity*2; i < valueCapacity*4; i++) { if ((i & 1) == 1) { - mutator.set(i, 1); + vector.set(i, 1); } } for (int i = 0; i < valueCapacity*4; i++) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2)) { - assertFalse("unexpected cleared bit at index: " + i, accessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, accessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); } } @@ -400,26 +397,24 @@ public void testReallocAfterVectorTransfer2() { TransferPair transferPair = vector.getTransferPair(allocator); transferPair.transfer(); final NullableBitVector toVector = (NullableBitVector)transferPair.getTo(); - final NullableBitVector.Accessor toAccessor = toVector.getAccessor(); - final NullableBitVector.Mutator toMutator = toVector.getMutator(); assertEquals(valueCapacity * 4, toVector.getValueCapacity()); /* realloc the toVector */ - toMutator.setSafe(valueCapacity * 4, 1, 1); + toVector.setSafe(valueCapacity * 4, 1, 1); for (int i = 0; i < toVector.getValueCapacity(); i++) { if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity*2) || (i == valueCapacity*4)) { - assertFalse("unexpected cleared bit at index: " + i, toAccessor.isNull(i)); + assertFalse("unexpected cleared bit at index: " + i, toVector.isNull(i)); } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } else { - assertTrue("unexpected set bit at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 56d22932764..4d844d6d3ca 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -71,14 +71,14 @@ public void testValuesWriteRead() { for (int i = 0; i < intValues.length; i++) { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale); values[i] = decimal; - decimalVector.getMutator().setSafe(i, decimal); + decimalVector.setSafe(i, decimal); } - decimalVector.getMutator().setValueCount(intValues.length); + decimalVector.setValueCount(intValues.length); for (int i = 0; i < intValues.length; i++) { - BigDecimal value = decimalVector.getAccessor().getObject(i); - assertEquals(values[i], value); + BigDecimal value = decimalVector.getObject(i); + assertEquals("unexpected data at index: " + i, values[i], value); } } } @@ -92,7 +92,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { boolean hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(0), 3); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { @@ -103,7 +103,7 @@ public void testBigDecimalDifferentScaleAndPrecision() { hasError = false; try { BigDecimal decimal = new BigDecimal(BigInteger.valueOf(12345), 2); - decimalVector.getMutator().setSafe(0, decimal); + decimalVector.setSafe(0, decimal); } catch (UnsupportedOperationException ue) { hasError = true; } finally { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 1185246f88c..46a2bafa002 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -135,9 +135,9 @@ public void testEncodeLargeVector() { // now run through the decoder and verify we get the original back try (ValueVector decoded = DictionaryEncoder.decode(encoded, dictionary)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getAccessor().getValueCount(), decoded.getAccessor().getValueCount()); + assertEquals(vector.getValueCount(), decoded.getValueCount()); for (int i = 0; i < count; ++i) { - assertEquals(vector.getAccessor().getObject(i), decoded.getAccessor().getObject(i)); + assertEquals(vector.getObject(i), decoded.getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 168deacd6a6..2af6cd53aa0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -57,7 +57,7 @@ public void testIntType() { vector.allocateNew(); for (int i = 0; i < 10; i++) { - vector.getMutator().setNotNull(i); + vector.setNotNull(i); nested.set(i * 2, i); nested.set(i * 2 + 1, i + 10); } @@ -81,17 +81,16 @@ public void testIntType() { public void testFloatTypeNullable() { try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - vector.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + vector.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - vector.getMutator().setValueCount(10); + vector.setValueCount(10); UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -115,24 +114,22 @@ public void testFloatTypeNullable() { @Test public void testNestedInList() { try (ListVector vector = ListVector.empty("list", allocator)) { - ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); - FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); NullableIntVector innerVector = (NullableIntVector) tuples.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - int position = mutator.startNewValue(i); + int position = vector.startNewValue(i); for (int j = 0; j < i % 7; j++) { - tupleMutator.setNotNull(position + j); + tuples.setNotNull(position + j); innerVector.set((position + j) * 2, j); innerVector.set((position + j) * 2 + 1, j + 1); } - mutator.endValue(i, i % 7); + vector.endValue(i, i % 7); } } - mutator.setValueCount(10); + vector.setValueCount(10); UnionListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { @@ -161,24 +158,24 @@ public void testTransferPair() { try (FixedSizeListVector from = new FixedSizeListVector("from", allocator, 2, null, null); FixedSizeListVector to = new FixedSizeListVector("to", allocator, 2, null, null)) { NullableFloat4Vector nested = (NullableFloat4Vector) from.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); - NullableFloat4Vector.Mutator mutator = nested.getMutator(); from.allocateNew(); for (int i = 0; i < 10; i++) { if (i % 2 == 0) { - from.getMutator().setNotNull(i); - mutator.set(i * 2, i + 0.1f); - mutator.set(i * 2 + 1, i + 10.1f); + from.setNotNull(i); + nested.set(i * 2, i + 0.1f); + nested.set(i * 2 + 1, i + 10.1f); } } - from.getMutator().setValueCount(10); + from.setValueCount(10); TransferPair pair = from.makeTransferPair(to); pair.copyValueSafe(0, 1); pair.copyValueSafe(2, 2); to.copyFromSafe(4, 3, from); - to.getMutator().setValueCount(10); + + to.setValueCount(10); UnionFixedSizeListReader reader = to.getReader(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 59e1646e86e..f6aa86a3008 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -85,7 +85,7 @@ public void testCopyFrom() throws Exception { for (int i = 0; i < 3; i++) { outVector.copyFrom(i, i, inVector); } - outVector.getMutator().setValueCount(3); + outVector.setValueCount(3); // assert the output vector is correct FieldReader reader = outVector.getReader(); @@ -95,11 +95,9 @@ public void testCopyFrom() throws Exception { reader.setPosition(2); Assert.assertTrue("shouldn't be null", reader.isSet()); - /* check the exact contents of vector */ - final ListVector.Accessor accessor = outVector.getAccessor(); /* index 0 */ - Object result = accessor.getObject(0); + Object result = outVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(1), (Long) resultSet.get(0)); @@ -107,11 +105,11 @@ public void testCopyFrom() throws Exception { assertEquals(new Long(3), (Long) resultSet.get(2)); /* index 1 */ - result = accessor.getObject(1); + result = outVector.getObject(1); assertNull(result); /* index 2 */ - result = accessor.getObject(2); + result = outVector.getObject(2); resultSet = (ArrayList) result; assertEquals(0, resultSet.size()); } @@ -128,46 +126,46 @@ public void testSetLastSetUsage() throws Exception { /* allocate memory */ listVector.allocateNew(); - /* get inner vectors; bitVector and offsetVector */ - List innerVectors = listVector.getFieldInnerVectors(); - BitVector bitVector = (BitVector) innerVectors.get(0); - UInt4Vector offsetVector = (UInt4Vector) innerVectors.get(1); + /* get inner buffers; validityBuffer and offsetBuffer */ + + ArrowBuf validityBuffer = listVector.getValidityBuffer(); + ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get the underlying data vector -- NullableBigIntVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); int index = 0; int offset = 0; - /* write [10, 11, 12] to the list vector at index */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(0, 1, 10); - dataVector.getMutator().setSafe(1, 1, 11); - dataVector.getMutator().setSafe(2, 1, 12); - offsetVector.getMutator().setSafe(index + 1, 3); + /* write [10, 11, 12] to the list vector at index 0 */ + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(0, 1, 10); + dataVector.setSafe(1, 1, 11); + dataVector.setSafe(2, 1, 12); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 3); index += 1; /* write [13, 14] to the list vector at index 1 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(3, 1, 13); - dataVector.getMutator().setSafe(4, 1, 14); - offsetVector.getMutator().setSafe(index + 1, 5); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(3, 1, 13); + dataVector.setSafe(4, 1, 14); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 5); index += 1; /* write [15, 16, 17] to the list vector at index 2 */ - bitVector.getMutator().setSafe(index, 1); - dataVector.getMutator().setSafe(5, 1, 15); - dataVector.getMutator().setSafe(6, 1, 16); - dataVector.getMutator().setSafe(7, 1, 17); - offsetVector.getMutator().setSafe(index + 1, 8); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + dataVector.setSafe(5, 1, 15); + dataVector.setSafe(6, 1, 16); + dataVector.setSafe(7, 1, 17); + offsetBuffer.setInt((index + 1) * ListVector.OFFSET_WIDTH, 8); /* check current lastSet */ - assertEquals(Integer.toString(0), Integer.toString(listVector.getMutator().getLastSet())); + assertEquals(Integer.toString(0), Integer.toString(listVector.getLastSet())); /* set lastset and arbitrary valuecount for list vector. * @@ -208,54 +206,50 @@ public void testSetLastSetUsage() throws Exception { * [15, 16, 17] * } */ - listVector.getMutator().setLastSet(3); - listVector.getMutator().setValueCount(10); - - /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); + listVector.setLastSet(3); + listVector.setValueCount(10); index = 0; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - Object actual = valueAccessor.getObject(offset); + Object actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); index++; - offset = offsetAccessor.get(index); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(8), Integer.toString(offset)); - actual = valueAccessor.getObject(offsetAccessor.get(index)); + actual = dataVector.getObject(offset); assertNull(actual); } } @@ -308,99 +302,97 @@ public void testSplitAndTransfer() throws Exception { listWriter.bigInt().writeBigInt(23); listWriter.endList(); - listVector.getMutator().setValueCount(5); + listVector.setValueCount(5); - assertEquals(5, listVector.getMutator().getLastSet()); + assertEquals(5, listVector.getLastSet()); - /* get offsetVector */ - UInt4Vector offsetVector = (UInt4Vector) listVector.getOffsetVector(); + /* get offset buffer */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* get dataVector */ NullableBigIntVector dataVector = (NullableBigIntVector) listVector.getDataVector(); /* check the vector output */ - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); - final ValueVector.Accessor valueAccessor = dataVector.getAccessor(); int index = 0; int offset = 0; Object actual = null; /* index 0 */ - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(0), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(10), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(11), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(12), (Long) actual); /* index 1 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(13), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(14), (Long) actual); /* index 2 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(15), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(16), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(17), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(18), (Long) actual); /* index 3 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(9), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(19), (Long) actual); /* index 4 */ index++; - assertFalse(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertFalse(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(10), Integer.toString(offset)); - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(20), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(21), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(22), (Long) actual); offset++; - actual = valueAccessor.getObject(offset); + actual = dataVector.getObject(offset); assertEquals(new Long(23), (Long) actual); /* index 5 */ index++; - assertTrue(listVector.getAccessor().isNull(index)); - offset = offsetAccessor.get(index); + assertTrue(listVector.isNull(index)); + offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(14), Integer.toString(offset)); /* do split and transfer */ @@ -423,26 +415,26 @@ public void testSplitAndTransfer() throws Exception { transferPair.splitAndTransfer(start, splitLength); /* get offsetVector of toVector */ - UInt4Vector offsetVector1 = (UInt4Vector) toVector.getOffsetVector(); - UInt4Vector.Accessor offsetAccessor1 = offsetVector1.getAccessor(); + final ArrowBuf toOffsetBuffer = toVector.getOffsetBuffer(); /* get dataVector of toVector */ NullableBigIntVector dataVector1 = (NullableBigIntVector) toVector.getDataVector(); - NullableBigIntVector.Accessor valueAccessor1 = dataVector1.getAccessor(); for (int i = 0; i < splitLength; i++) { - dataLength1 = offsetAccessor.get(start + i + 1) - offsetAccessor.get(start + i); - dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i); + dataLength1 = offsetBuffer.getInt((start + i + 1) * ListVector.OFFSET_WIDTH) - + offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + dataLength2 = toOffsetBuffer.getInt((i + 1) * ListVector.OFFSET_WIDTH) + - toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); assertEquals("Different data lengths at index: " + i + " and start: " + start, dataLength1, dataLength2); - offset1 = offsetAccessor.get(start + i); - offset2 = offsetAccessor1.get(i); + offset1 = offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); + offset2 = toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); for (int j = 0; j < dataLength1; j++) { assertEquals("Different data at indexes: " + offset1 + " and " + offset2, - valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2)); + dataVector.getObject(offset1), dataVector1.getObject(offset2)); offset1++; offset2++; @@ -506,15 +498,14 @@ public void testNestedListVector() throws Exception { listWriter.endList(); - assertEquals(2, listVector.getMutator().getLastSet()); + assertEquals(2, listVector.getLastSet()); - listVector.getMutator().setValueCount(2); + listVector.setValueCount(2); - final ListVector.Accessor accessor = listVector.getAccessor(); - assertEquals(2, accessor.getValueCount()); + assertEquals(2, listVector.getValueCount()); /* get listVector value at index 0 -- the value itself is a listvector */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList> resultSet = (ArrayList>) result; ArrayList list; @@ -534,7 +525,7 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(175), list.get(3)); /* get listVector value at index 1 -- the value itself is a listvector */ - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList>) result; assertEquals(3, resultSet.size()); /* 3 inner lists at index 1 */ @@ -555,17 +546,16 @@ public void testNestedListVector() throws Exception { assertEquals(new Long(35), list.get(2)); /* check underlying bitVector */ - assertFalse(accessor.isNull(0)); - assertFalse(accessor.isNull(1)); + assertFalse(listVector.isNull(0)); + assertFalse(listVector.isNull(1)); - /* check underlying offsetVector */ - UInt4Vector offsetVector = listVector.getOffsetVector(); - final UInt4Vector.Accessor offsetAccessor = offsetVector.getAccessor(); + /* check underlying offsets */ + final ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); /* listVector has 2 lists at index 0 and 3 lists at index 1 */ - assertEquals(0, offsetAccessor.get(0)); - assertEquals(2, offsetAccessor.get(1)); - assertEquals(5, offsetAccessor.get(2)); + assertEquals(0, offsetBuffer.getInt(0 * ListVector.OFFSET_WIDTH)); + assertEquals(2, offsetBuffer.getInt(1 * ListVector.OFFSET_WIDTH)); + assertEquals(5, offsetBuffer.getInt(2 * ListVector.OFFSET_WIDTH)); } } @@ -591,17 +581,15 @@ public void testGetBufferAddress() throws Exception { listWriter.bigInt().writeBigInt(300); listWriter.endList(); - final ListVector.Accessor accessor = listVector.getAccessor(); - /* check listVector contents */ - Object result = accessor.getObject(0); + Object result = listVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); assertEquals(new Long(50), resultSet.get(0)); assertEquals(new Long(100), resultSet.get(1)); assertEquals(new Long(200), resultSet.get(2)); - result = accessor.getObject(1); + result = listVector.getObject(1); resultSet = (ArrayList) result; assertEquals(2, resultSet.size()); assertEquals(new Long(250), resultSet.get(0)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 86f0bf337f9..ede4f992291 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -64,26 +64,24 @@ public void testUnionVector() throws Exception { unionVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = unionVector.getMutator(); - mutator.setType(0, Types.MinorType.UINT4); - mutator.setSafe(0, uInt4Holder); - mutator.setType(2, Types.MinorType.UINT4); - mutator.setSafe(2, uInt4Holder); - mutator.setValueCount(4); + unionVector.setType(0, Types.MinorType.UINT4); + unionVector.setSafe(0, uInt4Holder); + unionVector.setType(2, Types.MinorType.UINT4); + unionVector.setSafe(2, uInt4Holder); + unionVector.setValueCount(4); // check that what we wrote is correct - final UnionVector.Accessor accessor = unionVector.getAccessor(); - assertEquals(4, accessor.getValueCount()); + assertEquals(4, unionVector.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(100, accessor.getObject(0)); + assertEquals(false, unionVector.isNull(0)); + assertEquals(100, unionVector.getObject(0)); - assertEquals(true, accessor.isNull(1)); + assertEquals(true, unionVector.isNull(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(100, accessor.getObject(2)); + assertEquals(false, unionVector.isNull(2)); + assertEquals(100, unionVector.getObject(2)); - assertEquals(true, accessor.isNull(3)); + assertEquals(true, unionVector.isNull(3)); } } @@ -93,16 +91,15 @@ public void testTransfer() throws Exception { srcVector.allocateNew(); // write some data - final UnionVector.Mutator mutator = srcVector.getMutator(); - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.BIT); - mutator.setSafe(1, newBitHolder(false)); - mutator.setType(3, MinorType.INT); - mutator.setSafe(3, newIntHolder(10)); - mutator.setType(5, MinorType.BIT); - mutator.setSafe(5, newBitHolder(false)); - mutator.setValueCount(6); + srcVector.setType(0, MinorType.INT); + srcVector.setSafe(0, newIntHolder(5)); + srcVector.setType(1, MinorType.BIT); + srcVector.setSafe(1, newBitHolder(false)); + srcVector.setType(3, MinorType.INT); + srcVector.setSafe(3, newIntHolder(10)); + srcVector.setType(5, MinorType.BIT); + srcVector.setSafe(5, newBitHolder(false)); + srcVector.setValueCount(6); try (UnionVector destVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { TransferPair pair = srcVector.makeTransferPair(destVector); @@ -116,9 +113,9 @@ public void testTransfer() throws Exception { assertEquals(srcVector.getField(), destVector.getField()); // now check the values are transferred - assertEquals(srcVector.getAccessor().getValueCount(), destVector.getAccessor().getValueCount()); - for (int i = 0; i < srcVector.getAccessor().getValueCount(); i++) { - assertEquals("Different values at index " + i, srcVector.getAccessor().get(i), destVector.getAccessor().get(i)); + assertEquals(srcVector.getValueCount(), destVector.getValueCount()); + for (int i = 0; i < srcVector.getValueCount(); i++) { + assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i)); } } } @@ -127,61 +124,58 @@ public void testTransfer() throws Exception { @Test public void testSplitAndTransfer() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.INT); - sourceMutator.setSafe(1, newIntHolder(10)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(15)); - sourceMutator.setType(3, MinorType.INT); - sourceMutator.setSafe(3, newIntHolder(20)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(25)); - sourceMutator.setType(5, MinorType.INT); - sourceMutator.setSafe(5, newIntHolder(30)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(35)); - sourceMutator.setType(7, MinorType.INT); - sourceMutator.setSafe(7, newIntHolder(40)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(45)); - sourceMutator.setType(9, MinorType.INT); - sourceMutator.setSafe(9, newIntHolder(50)); - sourceMutator.setValueCount(10); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); + sourceVector.setType(1, MinorType.INT); + sourceVector.setSafe(1, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(15)); + sourceVector.setType(3, MinorType.INT); + sourceVector.setSafe(3, newIntHolder(20)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(25)); + sourceVector.setType(5, MinorType.INT); + sourceVector.setSafe(5, newIntHolder(30)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(35)); + sourceVector.setType(7, MinorType.INT); + sourceVector.setSafe(7, newIntHolder(40)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(45)); + sourceVector.setType(9, MinorType.INT); + sourceVector.setSafe(9, newIntHolder(50)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(10, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(15, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(20, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(25, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(30, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(35, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(40, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(45, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(50, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(10, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(15, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(20, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(25, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(30, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(35, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(40, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(45, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(50, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 3}, {3, 1}, @@ -199,8 +193,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceAccessor.getObject(start + i), - toAccessor.getObject(i)); + assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i), + toVector.getObject(i)); } } } @@ -210,70 +204,67 @@ public void testSplitAndTransfer() throws Exception { @Test public void testSplitAndTransferWithMixedVectors() throws Exception { try (UnionVector sourceVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator sourceMutator = sourceVector.getMutator(); - final UnionVector.Accessor sourceAccessor = sourceVector.getAccessor(); sourceVector.allocateNew(); /* populate the UnionVector */ - sourceMutator.setType(0, MinorType.INT); - sourceMutator.setSafe(0, newIntHolder(5)); + sourceVector.setType(0, MinorType.INT); + sourceVector.setSafe(0, newIntHolder(5)); - sourceMutator.setType(1, MinorType.FLOAT4); - sourceMutator.setSafe(1, newFloat4Holder(5.5f)); + sourceVector.setType(1, MinorType.FLOAT4); + sourceVector.setSafe(1, newFloat4Holder(5.5f)); - sourceMutator.setType(2, MinorType.INT); - sourceMutator.setSafe(2, newIntHolder(10)); + sourceVector.setType(2, MinorType.INT); + sourceVector.setSafe(2, newIntHolder(10)); - sourceMutator.setType(3, MinorType.FLOAT4); - sourceMutator.setSafe(3, newFloat4Holder(10.5f)); + sourceVector.setType(3, MinorType.FLOAT4); + sourceVector.setSafe(3, newFloat4Holder(10.5f)); - sourceMutator.setType(4, MinorType.INT); - sourceMutator.setSafe(4, newIntHolder(15)); + sourceVector.setType(4, MinorType.INT); + sourceVector.setSafe(4, newIntHolder(15)); - sourceMutator.setType(5, MinorType.FLOAT4); - sourceMutator.setSafe(5, newFloat4Holder(15.5f)); + sourceVector.setType(5, MinorType.FLOAT4); + sourceVector.setSafe(5, newFloat4Holder(15.5f)); - sourceMutator.setType(6, MinorType.INT); - sourceMutator.setSafe(6, newIntHolder(20)); + sourceVector.setType(6, MinorType.INT); + sourceVector.setSafe(6, newIntHolder(20)); - sourceMutator.setType(7, MinorType.FLOAT4); - sourceMutator.setSafe(7, newFloat4Holder(20.5f)); + sourceVector.setType(7, MinorType.FLOAT4); + sourceVector.setSafe(7, newFloat4Holder(20.5f)); - sourceMutator.setType(8, MinorType.INT); - sourceMutator.setSafe(8, newIntHolder(30)); + sourceVector.setType(8, MinorType.INT); + sourceVector.setSafe(8, newIntHolder(30)); - sourceMutator.setType(9, MinorType.FLOAT4); - sourceMutator.setSafe(9, newFloat4Holder(30.5f)); - sourceMutator.setValueCount(10); + sourceVector.setType(9, MinorType.FLOAT4); + sourceVector.setSafe(9, newFloat4Holder(30.5f)); + sourceVector.setValueCount(10); /* check the vector output */ - assertEquals(10, sourceAccessor.getValueCount()); - assertEquals(false, sourceAccessor.isNull(0)); - assertEquals(5, sourceAccessor.getObject(0)); - assertEquals(false, sourceAccessor.isNull(1)); - assertEquals(5.5f, sourceAccessor.getObject(1)); - assertEquals(false, sourceAccessor.isNull(2)); - assertEquals(10, sourceAccessor.getObject(2)); - assertEquals(false, sourceAccessor.isNull(3)); - assertEquals(10.5f, sourceAccessor.getObject(3)); - assertEquals(false, sourceAccessor.isNull(4)); - assertEquals(15, sourceAccessor.getObject(4)); - assertEquals(false, sourceAccessor.isNull(5)); - assertEquals(15.5f, sourceAccessor.getObject(5)); - assertEquals(false, sourceAccessor.isNull(6)); - assertEquals(20, sourceAccessor.getObject(6)); - assertEquals(false, sourceAccessor.isNull(7)); - assertEquals(20.5f, sourceAccessor.getObject(7)); - assertEquals(false, sourceAccessor.isNull(8)); - assertEquals(30, sourceAccessor.getObject(8)); - assertEquals(false, sourceAccessor.isNull(9)); - assertEquals(30.5f, sourceAccessor.getObject(9)); + assertEquals(10, sourceVector.getValueCount()); + assertEquals(false, sourceVector.isNull(0)); + assertEquals(5, sourceVector.getObject(0)); + assertEquals(false, sourceVector.isNull(1)); + assertEquals(5.5f, sourceVector.getObject(1)); + assertEquals(false, sourceVector.isNull(2)); + assertEquals(10, sourceVector.getObject(2)); + assertEquals(false, sourceVector.isNull(3)); + assertEquals(10.5f, sourceVector.getObject(3)); + assertEquals(false, sourceVector.isNull(4)); + assertEquals(15, sourceVector.getObject(4)); + assertEquals(false, sourceVector.isNull(5)); + assertEquals(15.5f, sourceVector.getObject(5)); + assertEquals(false, sourceVector.isNull(6)); + assertEquals(20, sourceVector.getObject(6)); + assertEquals(false, sourceVector.isNull(7)); + assertEquals(20.5f, sourceVector.getObject(7)); + assertEquals(false, sourceVector.isNull(8)); + assertEquals(30, sourceVector.getObject(8)); + assertEquals(false, sourceVector.isNull(9)); + assertEquals(30.5f, sourceVector.getObject(9)); try (UnionVector toVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { final TransferPair transferPair = sourceVector.makeTransferPair(toVector); - final UnionVector.Accessor toAccessor = toVector.getAccessor(); final int[][] transferLengths = {{0, 2}, {2, 1}, @@ -290,7 +281,7 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different values at index: " + i, sourceAccessor.getObject(start + i), toAccessor.getObject(i)); + assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i)); } } } @@ -300,37 +291,35 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { @Test public void testGetBufferAddress() throws Exception { try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) { - final UnionVector.Mutator mutator = vector.getMutator(); - final UnionVector.Accessor accessor = vector.getAccessor(); boolean error = false; vector.allocateNew(); /* populate the UnionVector */ - mutator.setType(0, MinorType.INT); - mutator.setSafe(0, newIntHolder(5)); + vector.setType(0, MinorType.INT); + vector.setSafe(0, newIntHolder(5)); - mutator.setType(1, MinorType.FLOAT4); - mutator.setSafe(1, newFloat4Holder(5.5f)); + vector.setType(1, MinorType.FLOAT4); + vector.setSafe(1, newFloat4Holder(5.5f)); - mutator.setType(2, MinorType.INT); - mutator.setSafe(2, newIntHolder(10)); + vector.setType(2, MinorType.INT); + vector.setSafe(2, newIntHolder(10)); - mutator.setType(3, MinorType.FLOAT4); - mutator.setSafe(3, newFloat4Holder(10.5f)); + vector.setType(3, MinorType.FLOAT4); + vector.setSafe(3, newFloat4Holder(10.5f)); - mutator.setValueCount(10); + vector.setValueCount(10); /* check the vector output */ - assertEquals(10, accessor.getValueCount()); - assertEquals(false, accessor.isNull(0)); - assertEquals(5, accessor.getObject(0)); - assertEquals(false, accessor.isNull(1)); - assertEquals(5.5f, accessor.getObject(1)); - assertEquals(false, accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); - assertEquals(false, accessor.isNull(3)); - assertEquals(10.5f, accessor.getObject(3)); + assertEquals(10, vector.getValueCount()); + assertEquals(false, vector.isNull(0)); + assertEquals(5, vector.getObject(0)); + assertEquals(false, vector.isNull(1)); + assertEquals(5.5f, vector.getObject(1)); + assertEquals(false, vector.isNull(2)); + assertEquals(10, vector.getObject(2)); + assertEquals(false, vector.isNull(3)); + assertEquals(10.5f, vector.getObject(3)); List buffers = vector.getFieldBuffers(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index b7f88c3babf..c7ee202f946 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -487,8 +487,6 @@ public void testNullableFixedType1() { // Create a new value vector for 1024 integers. try (final NullableUInt4Vector vector = newVector(NullableUInt4Vector.class, EMPTY_SCHEMA_PATH, new ArrowType.Int(32, false), allocator);) { - final NullableUInt4Vector.Mutator mutator = vector.getMutator(); - final NullableUInt4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 1024; @@ -500,33 +498,33 @@ public void testNullableFixedType1() { assertEquals(initialCapacity, vector.getValueCapacity()); // Put and set a few values - mutator.set(0, 100); - mutator.set(1, 101); - mutator.set(100, 102); - mutator.set(1022, 103); - mutator.set(1023, 104); + vector.set(0, 100); + vector.set(1, 101); + vector.set(100, 102); + vector.set(1022, 103); + vector.set(1023, 104); /* check vector contents */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); int val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } try { - mutator.set(1024, 10000); + vector.set(1024, 10000); } catch (IndexOutOfBoundsException ie) { error = true; @@ -537,7 +535,7 @@ public void testNullableFixedType1() { } try { - accessor.get(1024); + vector.get(1024); } catch (IndexOutOfBoundsException ie) { error = true; @@ -548,28 +546,28 @@ public void testNullableFixedType1() { } /* should trigger a realloc of the underlying bitvector and valuevector */ - mutator.setSafe(1024, 10000); + vector.setSafe(1024, 10000); /* check new capacity */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector contents should still be intact after realloc */ - assertEquals(100, accessor.get(0)); - assertEquals(101, accessor.get(1)); - assertEquals(102, accessor.get(100)); - assertEquals(103, accessor.get(1022)); - assertEquals(104, accessor.get(1023)); - assertEquals(10000, accessor.get(1024)); + assertEquals(100, vector.get(0)); + assertEquals(101, vector.get(1)); + assertEquals(102, vector.get(100)); + assertEquals(103, vector.get(1022)); + assertEquals(104, vector.get(1023)); + assertEquals(10000, vector.get(1024)); val = 0; /* check unset bits/null values */ for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) { if (i <= 99) { - assertTrue(accessor.isNull(i)); + assertTrue(vector.isNull(i)); } if(j <= 1021) { - assertTrue(accessor.isNull(j)); + assertTrue(vector.isNull(j)); } } @@ -581,7 +579,7 @@ public void testNullableFixedType1() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -590,8 +588,6 @@ public void testNullableFixedType1() { public void testNullableFixedType2() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator);) { - final NullableFloat4Vector.Mutator mutator = vector.getMutator(); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); boolean error = false; int initialCapacity = 16; @@ -603,17 +599,17 @@ public void testNullableFixedType2() { assertEquals(initialCapacity, vector.getValueCapacity()); /* populate the vector */ - mutator.set(0, 100.5f); - mutator.set(2, 201.5f); - mutator.set(4, 300.3f); - mutator.set(6, 423.8f); - mutator.set(8, 555.6f); - mutator.set(10, 66.6f); - mutator.set(12, 78.8f); - mutator.set(14, 89.5f); + vector.set(0, 100.5f); + vector.set(2, 201.5f); + vector.set(4, 300.3f); + vector.set(6, 423.8f); + vector.set(8, 555.6f); + vector.set(10, 66.6f); + vector.set(12, 78.8f); + vector.set(14, 89.5f); try { - mutator.set(16, 90.5f); + vector.set(16, 90.5f); } catch (IndexOutOfBoundsException ie) { error = true; @@ -624,25 +620,25 @@ public void testNullableFixedType2() { } /* check vector contents */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); try { - accessor.get(16); + vector.get(16); } catch (IndexOutOfBoundsException ie) { error = true; @@ -653,29 +649,28 @@ public void testNullableFixedType2() { } /* this should trigger a realloc() */ - mutator.setSafe(16, 90.5f); + vector.setSafe(16, 90.5f); /* underlying buffer should now be able to store double the number of values */ assertEquals(initialCapacity * 2, vector.getValueCapacity()); /* vector data should still be intact after realloc */ - assertEquals(100.5f, accessor.get(0), 0); - assertTrue(accessor.isNull(1)); - assertEquals(201.5f, accessor.get(2), 0); - assertTrue(accessor.isNull(3)); - assertEquals(300.3f, accessor.get(4), 0); - assertTrue(accessor.isNull(5)); - assertEquals(423.8f, accessor.get(6), 0); - assertTrue(accessor.isNull(7)); - assertEquals(555.6f, accessor.get(8), 0); - assertTrue(accessor.isNull(9)); - assertEquals(66.6f, accessor.get(10), 0); - assertTrue(accessor.isNull(11)); - assertEquals(78.8f, accessor.get(12), 0); - assertTrue(accessor.isNull(13)); - assertEquals(89.5f, accessor.get(14), 0); - assertTrue(accessor.isNull(15)); - assertEquals(90.5f, accessor.get(16), 0); + assertEquals(100.5f, vector.get(0), 0); + assertTrue(vector.isNull(1)); + assertEquals(201.5f, vector.get(2), 0); + assertTrue(vector.isNull(3)); + assertEquals(300.3f, vector.get(4), 0); + assertTrue(vector.isNull(5)); + assertEquals(423.8f, vector.get(6), 0); + assertTrue(vector.isNull(7)); + assertEquals(555.6f, vector.get(8), 0); + assertTrue(vector.isNull(9)); + assertEquals(66.6f, vector.get(10), 0); + assertTrue(vector.isNull(11)); + assertEquals(78.8f, vector.get(12), 0); + assertTrue(vector.isNull(13)); + assertEquals(89.5f, vector.get(14), 0); + assertTrue(vector.isNull(15)); /* reset the vector */ vector.reset(); @@ -685,7 +680,7 @@ public void testNullableFixedType2() { /* vector data should be zeroed out */ for(int i = 0; i < (initialCapacity * 2); i++) { - assertTrue("non-null data not expected at index: " + i, accessor.isNull(i)); + assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); } } } @@ -928,32 +923,30 @@ public void testNullableVarType2() { // Create a new value vector for 1024 integers. try (final NullableVarBinaryVector vector = newNullableVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableVarBinaryVector.Mutator m = vector.getMutator(); vector.allocateNew(1024 * 10, 1024); - m.set(0, STR1); - m.set(1, STR2); - m.set(2, STR3); - m.setSafe(3, STR3, 1, STR3.length - 1); - m.setSafe(4, STR3, 2, STR3.length - 2); + vector.set(0, STR1); + vector.set(1, STR2); + vector.set(2, STR3); + vector.setSafe(3, STR3, 1, STR3.length - 1); + vector.setSafe(4, STR3, 2, STR3.length - 2); ByteBuffer STR3ByteBuffer = ByteBuffer.wrap(STR3); - m.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); - m.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); + vector.setSafe(5, STR3ByteBuffer, 1, STR3.length - 1); + vector.setSafe(6, STR3ByteBuffer, 2, STR3.length - 2); // Check the sample strings. - final NullableVarBinaryVector.Accessor accessor = vector.getAccessor(); - assertArrayEquals(STR1, accessor.get(0)); - assertArrayEquals(STR2, accessor.get(1)); - assertArrayEquals(STR3, accessor.get(2)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(3)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(4)); - assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), accessor.get(5)); - assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), accessor.get(6)); + assertArrayEquals(STR1, vector.get(0)); + assertArrayEquals(STR2, vector.get(1)); + assertArrayEquals(STR3, vector.get(2)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(3)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(4)); + assertArrayEquals(Arrays.copyOfRange(STR3, 1, STR3.length), vector.get(5)); + assertArrayEquals(Arrays.copyOfRange(STR3, 2, STR3.length), vector.get(6)); // Ensure null value throws. boolean b = false; try { - vector.getAccessor().get(7); + vector.get(7); } catch (IllegalStateException e) { b = true; } finally { @@ -1064,8 +1057,6 @@ public void testReallocAfterVectorTransfer1() { @Test /* NullableFloat8Vector */ public void testReallocAfterVectorTransfer2() { try (final NullableFloat8Vector vector = new NullableFloat8Vector(EMPTY_SCHEMA_PATH, allocator)) { - final NullableFloat8Vector.Mutator mutator = vector.getMutator(); - final NullableFloat8Vector.Accessor accessor = vector.getAccessor(); final int initialDefaultCapacity = 4096; boolean error = false; @@ -1076,7 +1067,7 @@ public void testReallocAfterVectorTransfer2() { double baseValue = 100.375; for (int i = 0; i < initialDefaultCapacity; i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } /* the above setSafe calls should not have triggered a realloc as @@ -1085,33 +1076,33 @@ public void testReallocAfterVectorTransfer2() { assertEquals(initialDefaultCapacity, vector.getValueCapacity()); for (int i = 0; i < initialDefaultCapacity; i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); + vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity); assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity()); for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 2); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } /* this should trigger a realloc */ - mutator.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); + vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2)); assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity()); for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) { - mutator.setSafe(i, baseValue + (double)i); + vector.setSafe(i, baseValue + (double)i); } for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - double value = accessor.get(i); + double value = vector.get(i); assertEquals(baseValue + (double)i, value, 0); } @@ -1123,12 +1114,11 @@ public void testReallocAfterVectorTransfer2() { transferPair.transfer(); NullableFloat8Vector toVector = (NullableFloat8Vector)transferPair.getTo(); - final NullableFloat8Vector.Accessor toAccessor = toVector.getAccessor(); /* check toVector contents before realloc */ for (int i = 0; i < (initialDefaultCapacity * 4); i++) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } @@ -1138,12 +1128,12 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < (initialDefaultCapacity * 8); i++) { if (i < (initialDefaultCapacity * 4)) { - assertFalse("unexpected null value at index: " + i, toAccessor.isNull(i)); - double value = toAccessor.get(i); + assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + double value = toVector.get(i); assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0); } else { - assertTrue("unexpected non-null value at index: " + i, toAccessor.isNull(i)); + assertTrue("unexpected non-null value at index: " + i, toVector.isNull(i)); } } @@ -1351,32 +1341,30 @@ public void testReallocAfterVectorTransfer4() { public void testReAllocNullableFixedWidthVector() { // Create a new value vector for 1024 integers try (final NullableFloat4Vector vector = newVector(NullableFloat4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) { - final NullableFloat4Vector.Mutator m = vector.getMutator(); vector.allocateNew(1024); assertEquals(1024, vector.getValueCapacity()); // Put values in indexes that fall within the initial allocation - m.setSafe(0, 100.1f); - m.setSafe(100, 102.3f); - m.setSafe(1023, 104.5f); + vector.setSafe(0, 100.1f); + vector.setSafe(100, 102.3f); + vector.setSafe(1023, 104.5f); // Now try to put values in space that falls beyond the initial allocation - m.setSafe(2000, 105.5f); + vector.setSafe(2000, 105.5f); // Check valueCapacity is more than initial allocation assertEquals(1024 * 2, vector.getValueCapacity()); - final NullableFloat4Vector.Accessor accessor = vector.getAccessor(); - assertEquals(100.1f, accessor.get(0), 0); - assertEquals(102.3f, accessor.get(100), 0); - assertEquals(104.5f, accessor.get(1023), 0); - assertEquals(105.5f, accessor.get(2000), 0); + assertEquals(100.1f, vector.get(0), 0); + assertEquals(102.3f, vector.get(100), 0); + assertEquals(104.5f, vector.get(1023), 0); + assertEquals(105.5f, vector.get(2000), 0); // Set the valueCount to be more than valueCapacity of current allocation. This is possible for NullableValueVectors // as we don't call setSafe for null values, but we do call setValueCount when all values are inserted into the // vector - m.setValueCount(vector.getValueCapacity() + 200); + vector.setValueCount(vector.getValueCapacity() + 200); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 531a46c2734..293ffbfe192 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -28,6 +28,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; import org.junit.Assert; @@ -115,7 +116,7 @@ public void testListType() { vector.reAlloc(); assertEquals(2047, vector.getValueCapacity()); // note: size - 1 - assertEquals(0, vector.getOffsetVector().getAccessor().get(2014)); + assertEquals(0, vector.getOffsetBuffer().getInt(2014 * ListVector.OFFSET_WIDTH)); } } @@ -138,7 +139,7 @@ public void testMapType() { vector.reAlloc(); assertEquals(1024, vector.getValueCapacity()); - assertNull(vector.getAccessor().getObject(513)); + assertNull(vector.getObject(513)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 0b7928dca2c..3853eecac05 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -199,8 +199,10 @@ public void testLoadValidityBuffer() throws IOException { int count = 10; ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i+=2) { - ArrowBuf buf1 = allocator.buffer((int)Math.ceil(count / 8.0)); + ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); ArrowBuf buf2 = allocator.buffer(count * 4); // integers + buf1.setZero(0, buf1.capacity()); + buf2.setZero(0, buf2.capacity()); values[i] = buf1; values[i+1] = buf2; for (int j = 0; j < count; j++) { @@ -314,7 +316,7 @@ public void testUnloadLoadDuplicates() throws IOException { public static VectorUnloader newVectorUnloader(FieldVector root) { Schema schema = new Schema(root.getField().getChildren()); - int valueCount = root.getAccessor().getValueCount(); + int valueCount = root.getValueCount(); List fields = root.getChildrenFromFields(); VectorSchemaRoot vsr = new VectorSchemaRoot(schema.getFields(), fields, valueCount); return new VectorUnloader(vsr); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 97efb7d5a6d..b0d6cf555e9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -80,24 +80,23 @@ public void testPromoteToUnion() throws Exception { writer.end(); - container.getMutator().setValueCount(5); + container.setValueCount(5); final UnionVector uv = v.getChild("A", UnionVector.class); - final UnionVector.Accessor accessor = uv.getAccessor(); - assertFalse("0 shouldn't be null", accessor.isNull(0)); - assertEquals(false, accessor.getObject(0)); + assertFalse("0 shouldn't be null", uv.isNull(0)); + assertEquals(false, uv.getObject(0)); - assertFalse("1 shouldn't be null", accessor.isNull(1)); - assertEquals(true, accessor.getObject(1)); + assertFalse("1 shouldn't be null", uv.isNull(1)); + assertEquals(true, uv.getObject(1)); - assertFalse("2 shouldn't be null", accessor.isNull(2)); - assertEquals(10, accessor.getObject(2)); + assertFalse("2 shouldn't be null", uv.isNull(2)); + assertEquals(10, uv.getObject(2)); - assertTrue("3 should be null", accessor.isNull(3)); + assertTrue("3 should be null", uv.isNull(3)); - assertFalse("4 shouldn't be null", accessor.isNull(4)); - assertEquals(100, accessor.getObject(4)); + assertFalse("4 shouldn't be null", uv.isNull(4)); + assertEquals(100, uv.getObject(4)); container.clear(); container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 856d60724b0..52defdc2e1b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -456,7 +456,7 @@ public void simpleUnion() { unionWriter.writeFloat4((float) i); } } - vector.getMutator().setValueCount(COUNT); + vector.setValueCount(COUNT); UnionReader unionReader = new UnionReader(vector); for (int i = 0; i < COUNT; i++) { unionReader.setPosition(i); @@ -834,7 +834,7 @@ public void complexCopierWithList() { TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); - JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getAccessor().getObject(0); + JsonStringHashMap toMapValue = (JsonStringHashMap) toMapVector.getObject(0); JsonStringArrayList object = (JsonStringArrayList) toMapValue.get("list"); assertEquals(1, object.get(0)); assertEquals(2, object.get(1)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 60009b0a463..874ba99e20f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -97,14 +97,8 @@ public void tearDown() { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - FieldVector fv = root.getVector("int"); - if (fv instanceof NullableIntVector) { - Assert.assertEquals(i, fv.getObject(i)); - Assert.assertEquals(Integer.valueOf(i), fv.getObject(i)); - } else { - Assert.assertEquals(i, fv.getAccessor().getObject(i)); - Assert.assertEquals(Long.valueOf(i), fv.getAccessor().getObject(i)); - } + Assert.assertEquals(i, root.getVector("int").getObject(i)); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); } } @@ -146,10 +140,9 @@ protected void writeComplexData(int count, MapVector parent) { public void printVectors(List vectors) { for (FieldVector vector : vectors) { LOGGER.debug(vector.getField().getName()); - Accessor accessor = vector.getAccessor(); - int valueCount = accessor.getValueCount(); + int valueCount = vector.getValueCount(); for (int i = 0; i < valueCount; i++) { - LOGGER.debug(String.valueOf(accessor.getObject(i))); + LOGGER.debug(String.valueOf(vector.getObject(i))); } } } @@ -159,14 +152,14 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - Object intVal = root.getVector("int").getAccessor().getObject(i); + Object intVal = root.getVector("int").getObject(i); if (i % 5 != 3) { Assert.assertEquals(i, intVal); } else { Assert.assertNull(intVal); } - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i)); - Assert.assertEquals(i % 3, ((List) root.getVector("list").getAccessor().getObject(i)).size()); + Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + Assert.assertEquals(i % 3, ((List) root.getVector("list").getObject(i)).size()); NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); FieldReader mapReader = root.getVector("map").getReader(); mapReader.setPosition(i); @@ -209,15 +202,15 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { Assert.assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { - long dateVal = ((NullableDateMilliVector) root.getVector("date")).getAccessor().get(i); + long dateVal = ((NullableDateMilliVector) root.getVector("date")).get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); - long timeVal = ((NullableTimeMilliVector) root.getVector("time")).getAccessor().get(i); + long timeVal = ((NullableTimeMilliVector) root.getVector("time")).get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); - Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); + Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i); Assert.assertEquals(dt, timestampMilliVal); - Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i); + Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getObject(i); Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal); } } @@ -282,7 +275,7 @@ protected VectorSchemaRoot writeFlatDictionaryData(BufferAllocator bufferAllocat List fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(), encodedVector2.getField()); List vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2); - return new VectorSchemaRoot(fields, vectors, encodedVector1A.getAccessor().getValueCount()); + return new VectorSchemaRoot(fields, vectors, encodedVector1A.getValueCount()); } protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) { @@ -293,14 +286,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding1A); Assert.assertEquals(1L, encoding1A.getId()); - FieldVector.Accessor accessor = vector1A.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(0, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(null, accessor.getObject(2)); - Assert.assertEquals(2, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(2, accessor.getObject(5)); + Assert.assertEquals(6, vector1A.getValueCount()); + Assert.assertEquals(0, vector1A.getObject(0)); + Assert.assertEquals(1, vector1A.getObject(1)); + Assert.assertEquals(null, vector1A.getObject(2)); + Assert.assertEquals(2, vector1A.getObject(3)); + Assert.assertEquals(1, vector1A.getObject(4)); + Assert.assertEquals(2, vector1A.getObject(5)); FieldVector vector1B = root.getVector("varcharB"); Assert.assertNotNull(vector1B); @@ -310,14 +302,13 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertTrue(encoding1A.equals(encoding1B)); Assert.assertEquals(1L, encoding1B.getId()); - accessor = vector1B.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(2, accessor.getObject(0)); - Assert.assertEquals(1, accessor.getObject(1)); - Assert.assertEquals(2, accessor.getObject(2)); - Assert.assertEquals(null, accessor.getObject(3)); - Assert.assertEquals(1, accessor.getObject(4)); - Assert.assertEquals(0, accessor.getObject(5)); + Assert.assertEquals(6, vector1B.getValueCount()); + Assert.assertEquals(2, vector1B.getObject(0)); + Assert.assertEquals(1, vector1B.getObject(1)); + Assert.assertEquals(2, vector1B.getObject(2)); + Assert.assertEquals(null, vector1B.getObject(3)); + Assert.assertEquals(1, vector1B.getObject(4)); + Assert.assertEquals(0, vector1B.getObject(5)); FieldVector vector2 = root.getVector("sizes"); Assert.assertNotNull(vector2); @@ -326,30 +317,29 @@ protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider Assert.assertNotNull(encoding2); Assert.assertEquals(2L, encoding2.getId()); - accessor = vector2.getAccessor(); - Assert.assertEquals(6, accessor.getValueCount()); - Assert.assertEquals(null, accessor.getObject(0)); - Assert.assertEquals(2, accessor.getObject(1)); - Assert.assertEquals(1, accessor.getObject(2)); - Assert.assertEquals(1, accessor.getObject(3)); - Assert.assertEquals(2, accessor.getObject(4)); - Assert.assertEquals(null, accessor.getObject(5)); + Assert.assertEquals(6, vector2.getValueCount()); + Assert.assertEquals(null, vector2.getObject(0)); + Assert.assertEquals(2, vector2.getObject(1)); + Assert.assertEquals(1, vector2.getObject(2)); + Assert.assertEquals(1, vector2.getObject(3)); + Assert.assertEquals(2, vector2.getObject(4)); + Assert.assertEquals(null, vector2.getObject(5)); Dictionary dictionary1 = provider.lookup(1L); Assert.assertNotNull(dictionary1); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary1.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryAccessor.getObject(2)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary1.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); Dictionary dictionary2 = provider.lookup(2L); Assert.assertNotNull(dictionary2); - dictionaryAccessor = ((NullableVarCharVector) dictionary2.getVector()).getAccessor(); - Assert.assertEquals(3, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("micro"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("small"), dictionaryAccessor.getObject(1)); - Assert.assertEquals(new Text("large"), dictionaryAccessor.getObject(2)); + dictionaryVector = ((NullableVarCharVector) dictionary2.getVector()); + Assert.assertEquals(3, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1)); + Assert.assertEquals(new Text("large"), dictionaryVector.getObject(2)); } protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAllocator, DictionaryProvider.MapDictionaryProvider provider) { @@ -359,7 +349,7 @@ protected VectorSchemaRoot writeNestedDictionaryData(BufferAllocator bufferAlloc dictionaryVector.allocateNewSafe(); dictionaryVector.set(0, "foo".getBytes(StandardCharsets.UTF_8)); dictionaryVector.set(1, "bar".getBytes(StandardCharsets.UTF_8)); - dictionaryVector.getMutator().setValueCount(2); + dictionaryVector.setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(2L, false, null)); provider.put(dictionary); @@ -398,18 +388,17 @@ protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvide Assert.assertEquals(2L, encoding.getId()); Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType()); - FieldVector.Accessor accessor = vector.getAccessor(); - Assert.assertEquals(3, accessor.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), accessor.getObject(0)); - Assert.assertEquals(Arrays.asList(0), accessor.getObject(1)); - Assert.assertEquals(Arrays.asList(1), accessor.getObject(2)); + Assert.assertEquals(3, vector.getValueCount()); + Assert.assertEquals(Arrays.asList(0, 1), vector.getObject(0)); + Assert.assertEquals(Arrays.asList(0), vector.getObject(1)); + Assert.assertEquals(Arrays.asList(1), vector.getObject(2)); Dictionary dictionary = provider.lookup(2L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor(); - Assert.assertEquals(2, dictionaryAccessor.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); + NullableVarCharVector dictionaryVector = ((NullableVarCharVector) dictionary.getVector()); + Assert.assertEquals(2, dictionaryVector.getValueCount()); + Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { @@ -423,14 +412,14 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { decimalVector3.allocateNew(count); for (int i = 0; i < count; i++) { - decimalVector1.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); - decimalVector2.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); - decimalVector3.getMutator().setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); + decimalVector1.setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); + decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); + decimalVector3.setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); } - decimalVector1.getMutator().setValueCount(count); - decimalVector2.getMutator().setValueCount(count); - decimalVector3.getMutator().setValueCount(count); + decimalVector1.setValueCount(count); + decimalVector2.setValueCount(count); + decimalVector3.setValueCount(count); List fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(), decimalVector3.getField()); List vectors = ImmutableList.of(decimalVector1, decimalVector2, decimalVector3); @@ -446,19 +435,19 @@ protected void validateDecimalData(VectorSchemaRoot root) { for (int i = 0; i < count; i++) { // Verify decimal 1 vector - BigDecimal readValue = decimalVector1.getAccessor().getObject(i); + BigDecimal readValue = decimalVector1.getObject(i); ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType(); BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 2 vector - readValue = decimalVector2.getAccessor().getObject(i); + readValue = decimalVector2.getObject(i); type = (ArrowType.Decimal) decimalVector2.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale()); Assert.assertEquals(genValue, readValue); // Verify decimal 3 vector - readValue = decimalVector3.getAccessor().getObject(i); + readValue = decimalVector3.getObject(i); type = (ArrowType.Decimal) decimalVector3.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale()); Assert.assertEquals(genValue, readValue); @@ -571,7 +560,7 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { int numVarBinaryValues = 0; for (int i = 0; i < count; i++) { expectedArray[i] = (byte) i; - Object obj = listVector.getAccessor().getObject(i); + Object obj = listVector.getObject(i); List objList = (List) obj; if (i % 3 == 0) { Assert.assertTrue(objList.isEmpty()); @@ -586,10 +575,10 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { } // ListVector lastSet should be the index of last value + 1 - Assert.assertEquals(listVector.getMutator().getLastSet(), count); + Assert.assertEquals(listVector.getLastSet(), count); // NullableVarBinaryVector lastSet should be the index of last value NullableVarBinaryVector binaryVector = (NullableVarBinaryVector) listVector.getChildrenFromFields().get(0); - Assert.assertEquals(binaryVector.getMutator().getLastSet(), numVarBinaryValues - 1); + Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index feae08ea223..8559969a204 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -317,11 +317,11 @@ public void testWriteReadTiny() throws IOException { try (VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); // write file @@ -368,9 +368,9 @@ private void validateTinyData(VectorSchemaRoot root) { NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } @@ -397,7 +397,7 @@ public void testWriteReadMetadata() throws IOException { try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); - vector.getMutator().setValueCount(0); + vector.setValueCount(0); List vectors = ImmutableList.of(vector); VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0); @@ -578,13 +578,13 @@ public void testWriteReadFixedSizeList() throws IOException { parent.allocateNew(); for (int i = 0; i < 10; i++) { - tuples.getMutator().setNotNull(i); - floats.getMutator().set(i * 2, i + 0.1f); - floats.getMutator().set(i * 2 + 1, i + 10.1f); + tuples.setNotNull(i); + floats.set(i * 2, i + 0.1f); + floats.set(i * 2 + 1, i + 10.1f); ints.set(i, i); } - parent.getMutator().setValueCount(10); + parent.setValueCount(10); write(parent, file, stream); } @@ -600,8 +600,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadRecordBatch(rbBlock); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } @@ -616,8 +616,8 @@ public void testWriteReadFixedSizeList() throws IOException { arrowReader.loadNextBatch(); Assert.assertEquals(count, root.getRowCount()); for (int i = 0; i < 10; i++) { - Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getAccessor().getObject(i)); - Assert.assertEquals(i, root.getVector("ints").getAccessor().getObject(i)); + Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i)); + Assert.assertEquals(i, root.getVector("ints").getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java index e2efabef009..c7e34191a3f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStream.java @@ -70,11 +70,11 @@ public void testReadWrite() throws IOException { int numBatches = 1; root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector)root.getFieldVectors().get(0); for (int i = 0; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java index 40716942f02..f393733cc90 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowStreamPipe.java @@ -62,13 +62,13 @@ public void run() { writer.start(); for (int j = 0; j < numBatches; j++) { root.getFieldVectors().get(0).allocateNew(); - NullableTinyIntVector.Mutator mutator = (NullableTinyIntVector.Mutator) root.getFieldVectors().get(0).getMutator(); + NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); // Send a changing batch id first - mutator.set(0, j); + vector.set(0, j); for (int i = 1; i < 16; i++) { - mutator.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); + vector.set(i, i < 8 ? 1 : 0, (byte) (i + 1)); } - mutator.setValueCount(16); + vector.setValueCount(16); root.setRowCount(16); writer.writeBatch(); @@ -117,12 +117,12 @@ public boolean loadNextBatch() throws IOException { VectorSchemaRoot root = getVectorSchemaRoot(); Assert.assertEquals(16, root.getRowCount()); NullableTinyIntVector vector = (NullableTinyIntVector) root.getFieldVectors().get(0); - Assert.assertEquals((byte) (batchesRead - 1), vector.getAccessor().get(0)); + Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); for (int i = 1; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.getAccessor().get(i)); + Assert.assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.getAccessor().isNull(i)); + Assert.assertTrue(vector.isNull(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java index ee90d340d7c..5c4c48cd26b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java @@ -280,7 +280,7 @@ public void testSetStructLength() throws IOException { // initialize vectors try (VectorSchemaRoot root = reader.read();) { FieldVector vector = root.getVector("struct_nullable"); - Assert.assertEquals(7, vector.getAccessor().getValueCount()); + Assert.assertEquals(7, vector.getValueCount()); } } } From 0de6e69775277760b5df6185d3dc33208ba591fa Mon Sep 17 00:00:00 2001 From: Li Jin Date: Tue, 7 Nov 2017 11:12:57 -0500 Subject: [PATCH 03/10] ARROW-1717: [Java] Refactor JsonReader for new class hierarchy and fix integration tests --- .../BaseNullableVariableWidthVector.java | 46 -- .../arrow/vector/NullableBigIntVector.java | 37 +- .../arrow/vector/NullableDateDayVector.java | 34 -- .../arrow/vector/NullableDateMilliVector.java | 34 -- .../arrow/vector/NullableDecimalVector.java | 37 +- .../arrow/vector/NullableFloat4Vector.java | 37 +- .../arrow/vector/NullableFloat8Vector.java | 37 +- .../arrow/vector/NullableIntVector.java | 36 +- .../arrow/vector/NullableSmallIntVector.java | 37 +- .../arrow/vector/NullableTimeMicroVector.java | 34 -- .../arrow/vector/NullableTimeMilliVector.java | 34 -- .../arrow/vector/NullableTimeNanoVector.java | 34 -- .../arrow/vector/NullableTimeSecVector.java | 34 -- .../arrow/vector/NullableTimeStampVector.java | 36 -- .../arrow/vector/NullableTinyIntVector.java | 36 -- .../vector/file/json/JsonFileReader.java | 424 ++++++++++++------ 16 files changed, 290 insertions(+), 677 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index 6587cde633e..47daeee7f20 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -909,52 +909,6 @@ protected final void handleSafe(int index, int dataLength) { } } - - /****************************************************************** - * * - * helper methods currently * - * used by JsonFileReader and * - * JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Method used by Json Reader to explicitly set the data of the variable - * width vector elements. The method takes care of allocating the memory - * for the vector if caller hasn't done so. - * - * This method should not be used externally. - * - * @param data ArrowBuf for storing variable width elements in the vector - * @param offset offset of the element - * @param allocator memory allocator - * @param index position of the element in the vector - * @param value array of bytes for the element - * @param valueCount number of elements in the vector - * @return buffer holding the variable width data. - */ - public static ArrowBuf set(ArrowBuf data, ArrowBuf offset, - BufferAllocator allocator, int index, byte[] value, - int valueCount) { - if (data == null) { - data = allocator.buffer(INITIAL_BYTE_COUNT); - } - final int currentBufferCapacity = data.capacity(); - final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); - while (currentBufferCapacity < currentStartOffset + value.length) { - final ArrowBuf newBuf = allocator.buffer(currentBufferCapacity * 2); - newBuf.setBytes(0, data, 0, currentBufferCapacity); - data.release(); - data = newBuf; - } - data.setBytes(currentStartOffset, value, 0, value.length); - if (index == (valueCount - 1)) { - data.writerIndex(offset.getInt(valueCount * OFFSET_WIDTH)); - } - return data; - } - /** * Method used by Json Writer to read a variable width element from * the variable width vector and write to Json. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index ee40d708cc1..253427333a2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableBigIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; + public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** @@ -290,41 +290,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index 949287ecafe..e6b5b590f3f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index a0bdccedea4..8e15100f708 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -296,40 +296,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 8320f90830a..2a611c6a2d5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -37,7 +37,7 @@ * maintained to track which elements in the vector are null. */ public class NullableDecimalVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 16; + public static final byte TYPE_WIDTH = 16; private final FieldReader reader; private final int precision; @@ -355,41 +355,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value as array of bytes - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, byte[] value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - DecimalUtility.writeByteArrayToArrowBuf(value, buffer, index); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - - /****************************************************************** * * * vector transfer * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index 5b28065c402..3ba5cfcea52 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat4Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; + public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** @@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, float value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, float value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setFloat(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index 624abf2f272..2fb96a44fe7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat8Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; + public static final byte TYPE_WIDTH = 8; private final FieldReader reader; /** @@ -291,41 +291,6 @@ public void setSafe(int index, int isSet, double value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, double value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setDouble(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 6311daf4f5c..93deacbdfdc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; + public static final byte TYPE_WIDTH = 4; private final FieldReader reader; /** @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index c45a8d5f5bc..ed337188b5b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -34,7 +34,7 @@ * maintained to track which elements in the vector are null. */ public class NullableSmallIntVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 2; + public static final byte TYPE_WIDTH = 2; private final FieldReader reader; /** @@ -319,41 +319,6 @@ public void setSafe(int index, int isSet, short value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, short value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setShort(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index 454a4ac41a6..d7ea3222397 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -292,40 +292,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 8540d169242..07d8abba4e5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -294,40 +294,6 @@ public void setSafe(int index, int isSet, int value) { } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index 015226da223..947b2392f7a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index 2b2375e92f1..0a3cfaac047 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -291,40 +291,6 @@ public void setSafe(int index, int isSet, int value) { set(index, isSet, value); } - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setInt(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index b2a58bd4568..1bf2abc384d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -165,42 +165,6 @@ public void setSafe(int index, int isSet, long value) { set(index, isSet, value); } - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and * - * JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, long value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setLong(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index 15100530d0e..ccbfa32b85f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -318,42 +318,6 @@ public void setSafe(int index, int isSet, byte value) { set(index, isSet, value); } - - - /****************************************************************** - * * - * helper routines currently * - * used in JsonFileReader and JsonFileWriter * - * * - ******************************************************************/ - - - /** - * Given a data buffer, this method sets the element value at a particular - * position. Reallocates the buffer if needed. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param allocator allocator - * @param valueCount number of elements in the vector - * @param index position of the new element - * @param value element value - * @return data buffer - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, byte value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * TYPE_WIDTH); - } - buffer.setByte(index * TYPE_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * TYPE_WIDTH); - } - - return buffer; - } - /** * Given a data buffer, get the value stored at a particular position * in the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index b21fcaac391..d3c8b990394 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -35,16 +35,19 @@ import java.util.*; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.*; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.file.InvalidArrowFileException; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowVectorType; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.DictionaryUtility; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -186,6 +189,282 @@ public VectorSchemaRoot read() throws IOException { } } + private abstract class BufferReader { + abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException; + + final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { + readToken(START_ARRAY); + ArrowBuf buf = read(allocator, count); + readToken(END_ARRAY); + return buf; + } + } + + private class BufferHelper { + BufferReader BIT = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + final int bufferSize = BitVectorHelper.getValidityBufferSize(count); + ArrowBuf buf = allocator.buffer(bufferSize); + + // C++ integration test fails without this. + buf.setZero(0, bufferSize); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0); + } + + return buf; + } + }; + + BufferReader INT1 = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeByte(parser.getByteValue()); + } + + return buf; + } + }; + + BufferReader INT2 = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeShort(parser.getShortValue()); + } + + return buf; + } + }; + + BufferReader INT4 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeInt(parser.getIntValue()); + } + + return buf; + } + }; + + BufferReader INT8 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeLong(parser.getLongValue()); + } + + return buf; + } + }; + + BufferReader FLOAT4 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeFloat(parser.getFloatValue()); + } + + return buf; + } + }; + + BufferReader FLOAT8 = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + buf.writeDouble(parser.getDoubleValue()); + } + + return buf; + } + }; + + BufferReader DECIMAL = new BufferReader() { + + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = decodeHexSafe(parser.getValueAsString()); + DecimalUtility.writeByteArrayToArrowBuf(value, buf, i); + } + + return buf; + } + }; + + BufferReader VARCHAR = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrayList values = Lists.newArrayList(); + int bufferSize = 0; + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = parser.getValueAsString().getBytes(UTF_8); + values.add(value); + bufferSize += value.length; + + } + + ArrowBuf buf = allocator.buffer(bufferSize); + + for (byte[] value : values) { + buf.writeBytes(value); + } + + return buf; + } + }; + + BufferReader VARBINARY = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + ArrayList values = Lists.newArrayList(); + int bufferSize = 0; + for (int i = 0; i < count; i++) { + parser.nextToken(); + final byte[] value = decodeHexSafe(parser.readValueAs(String.class)); + values.add(value); + bufferSize += value.length; + + } + + ArrowBuf buf = allocator.buffer(bufferSize); + + for (byte[] value : values) { + buf.writeBytes(value); + } + + return buf; + } + }; + + } + + private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException { + ArrowBuf buf; + + BufferHelper helper = new BufferHelper(); + + BufferReader reader = null; + + if (bufferType.equals(VALIDITY)) { + reader = helper.BIT; + } else if (bufferType.equals(OFFSET)) { + reader = helper.INT4; + } else if (bufferType.equals(TYPE)) { + reader = helper.INT1; + } else if (bufferType.equals(DATA)) { + switch (type) { + case BIT: + reader = helper.BIT; + break; + case TINYINT: + reader = helper.INT1; + break; + case SMALLINT: + reader = helper.INT2; + break; + case INT: + reader = helper.INT4; + break; + case BIGINT: + reader = helper.INT8; + break; + case UINT1: + reader = helper.INT1; + break; + case UINT2: + reader = helper.INT2; + break; + case UINT4: + reader = helper.INT4; + break; + case UINT8: + reader = helper.INT8; + break; + case FLOAT4: + reader = helper.FLOAT4; + break; + case FLOAT8: + reader = helper.FLOAT8; + break; + case DECIMAL: + reader = helper.DECIMAL; + break; + case VARCHAR: + reader = helper.VARCHAR; + break; + case VARBINARY: + reader = helper.VARBINARY; + break; + case DATEDAY: + reader = helper.INT4; + break; + case DATEMILLI: + reader = helper.INT8; + break; + case TIMESEC: + case TIMEMILLI: + reader = helper.INT4; + break; + case TIMEMICRO: + case TIMENANO: + reader = helper.INT8; + break; + case TIMESTAMPNANO: + case TIMESTAMPMICRO: + case TIMESTAMPMILLI: + case TIMESTAMPSEC: + case TIMESTAMPNANOTZ: + case TIMESTAMPMICROTZ: + case TIMESTAMPMILLITZ: + case TIMESTAMPSECTZ: + reader = helper.INT8; + break; + default: + throw new UnsupportedOperationException("Cannot read array of type " + type); + } + } else { + throw new InvalidArrowFileException("Unrecognized buffer type " + bufferType); + } + + buf = reader.readBuffer(allocator, count); + assert buf != null; + return buf; + } + private void readFromJsonIntoVector(Field field, FieldVector vector) throws JsonParseException, IOException { List vectorTypes = field.getTypeLayout().getVectorTypes(); ArrowBuf[] vectorBuffers = new ArrowBuf[vectorTypes.size()]; @@ -212,7 +491,7 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name); } - /* Initialize the vector with required capacity but don't allocate since we would + /* Initialize the vector with required capacity but don't allocateNew since we would * be doing loadFieldBuffers. */ int valueCount = readNextField("count", Integer.class); @@ -221,29 +500,13 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); nextFieldIs(vectorType.getName()); - readToken(START_ARRAY); int innerBufferValueCount = valueCount; if (vectorType.equals(OFFSET)) { /* offset buffer has 1 additional value capacity */ innerBufferValueCount = valueCount + 1; } - for (int i = 0; i < innerBufferValueCount; i++) { - /* write data to the buffer */ - parser.nextToken(); - /* for variable width vectors, value count doesn't help pre-determining the capacity of - * the underlying data buffer. So we need to pass down the offset buffer (which was already - * populated in the previous iteration of this loop). - */ - if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR - || vector.getMinorType() == Types.MinorType.VARBINARY)) { - vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], - vectorBuffers[v-1], i, innerBufferValueCount); - } else { - vectorBuffers[v] = setValueFromParser(vectorType, vector, vectorBuffers[v], - null, i, innerBufferValueCount); - } - } - readToken(END_ARRAY); + + vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); } vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); @@ -253,7 +516,8 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json if (!fields.isEmpty()) { List vectorChildren = vector.getChildrenFromFields(); if (fields.size() != vectorChildren.size()) { - throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size()); + throw new IllegalArgumentException( + "fields and children are not the same size: " + fields.size() + " != " + vectorChildren.size()); } nextFieldIs("children"); readToken(START_ARRAY); @@ -280,126 +544,6 @@ private byte[] decodeHexSafe(String hexString) throws IOException { } } - private ArrowBuf setValueFromParser(ArrowVectorType bufferType, FieldVector vector, - ArrowBuf buffer, ArrowBuf offsetBuffer, int index, - int valueCount) throws IOException { - if (bufferType.equals(TYPE)) { - buffer = NullableTinyIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Byte.class)); - } else if (bufferType.equals(OFFSET)) { - buffer = BaseNullableVariableWidthVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - } else if (bufferType.equals(VALIDITY)) { - buffer = BitVectorHelper.setValidityBit(buffer, allocator, - valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); - } else if (bufferType.equals(DATA)) { - switch (vector.getMinorType()) { - case BIT: - buffer = BitVectorHelper.setValidityBit(buffer, allocator, - valueCount, index, parser.readValueAs(Boolean.class) ? 1 : 0); - break; - case TINYINT: - buffer = NullableTinyIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Byte.class)); - break; - case SMALLINT: - buffer = NullableSmallIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Short.class)); - break; - case INT: - buffer = NullableIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case BIGINT: - buffer = NullableBigIntVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case FLOAT4: - buffer = NullableFloat4Vector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Float.class)); - break; - case FLOAT8: - buffer = NullableFloat8Vector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Double.class)); - break; - case DECIMAL: - buffer = NullableDecimalVector.set(buffer, allocator, - valueCount, index, decodeHexSafe(parser.readValueAs(String.class))); - break; - case VARBINARY: - assert (offsetBuffer != null); - buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, - decodeHexSafe(parser.readValueAs(String.class)), valueCount); - break; - case VARCHAR: - assert (offsetBuffer != null); - buffer = BaseNullableVariableWidthVector.set(buffer, offsetBuffer, allocator, index, - parser.readValueAs(String.class).getBytes(UTF_8), valueCount); - break; - case DATEDAY: - buffer = NullableDateDayVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case DATEMILLI: - buffer = NullableDateMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESEC: - buffer = NullableTimeSecVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case TIMEMILLI: - buffer = NullableTimeMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Integer.class)); - break; - case TIMEMICRO: - buffer = NullableTimeMicroVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMENANO: - buffer = NullableTimeNanoVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSEC: - buffer = NullableTimeStampSecVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLI: - buffer = NullableTimeStampMilliVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICRO: - buffer = NullableTimeStampMicroVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANO: - buffer = NullableTimeStampNanoVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPSECTZ: - buffer = NullableTimeStampSecTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMILLITZ: - buffer = NullableTimeStampMilliTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPMICROTZ: - buffer = NullableTimeStampMicroTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - case TIMESTAMPNANOTZ: - buffer = NullableTimeStampNanoTZVector.set(buffer, allocator, - valueCount, index, parser.readValueAs(Long.class)); - break; - default: - throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); - } - } - - return buffer; - } - @Override public void close() throws IOException { parser.close(); From 2027f0bb60c4e60bad61a9e07add339c6e44528c Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 13 Nov 2017 17:20:56 -0800 Subject: [PATCH 04/10] ARROW-1476: [JAVA] Implement Final ValueVector Updates --- .../main/codegen/templates/UnionReader.java | 2 +- .../main/codegen/templates/UnionVector.java | 30 +- .../vector/BaseNullableFixedWidthVector.java | 61 ++-- .../BaseNullableVariableWidthVector.java | 308 ++++++++++++++-- .../org/apache/arrow/vector/BitVector.java | 4 +- .../arrow/vector/GenerateSampleData.java | 337 ++++++++++++++++++ .../arrow/vector/NullableBigIntVector.java | 4 +- .../arrow/vector/NullableBitVector.java | 4 +- .../arrow/vector/NullableDateDayVector.java | 4 +- .../arrow/vector/NullableDateMilliVector.java | 4 +- .../arrow/vector/NullableDecimalVector.java | 3 + .../arrow/vector/NullableFloat4Vector.java | 4 +- .../arrow/vector/NullableFloat8Vector.java | 4 +- .../arrow/vector/NullableIntVector.java | 4 +- .../vector/NullableIntervalDayVector.java | 2 + .../vector/NullableIntervalYearVector.java | 4 +- .../arrow/vector/NullableSmallIntVector.java | 4 +- .../arrow/vector/NullableTimeMicroVector.java | 4 +- .../arrow/vector/NullableTimeMilliVector.java | 2 + .../arrow/vector/NullableTimeNanoVector.java | 4 +- .../arrow/vector/NullableTimeSecVector.java | 4 +- .../NullableTimeStampMicroTZVector.java | 2 +- .../vector/NullableTimeStampMicroVector.java | 2 +- .../NullableTimeStampMilliTZVector.java | 2 +- .../vector/NullableTimeStampMilliVector.java | 2 +- .../vector/NullableTimeStampNanoTZVector.java | 2 +- .../vector/NullableTimeStampNanoVector.java | 2 +- .../vector/NullableTimeStampSecTZVector.java | 2 +- .../vector/NullableTimeStampSecVector.java | 2 +- .../arrow/vector/NullableTimeStampVector.java | 2 + .../arrow/vector/NullableTinyIntVector.java | 4 +- .../arrow/vector/NullableUInt1Vector.java | 4 +- .../arrow/vector/NullableUInt2Vector.java | 4 +- .../arrow/vector/NullableUInt4Vector.java | 4 +- .../arrow/vector/NullableUInt8Vector.java | 4 +- .../arrow/vector/NullableVarBinaryVector.java | 225 ++---------- .../arrow/vector/NullableVarCharVector.java | 225 ++---------- .../complex/BaseRepeatedValueVector.java | 10 +- .../vector/complex/FixedSizeListVector.java | 21 +- .../arrow/vector/complex/ListVector.java | 68 +++- .../vector/complex/NullableMapVector.java | 31 +- .../apache/arrow/vector/TestUnionVector.java | 22 +- 42 files changed, 904 insertions(+), 533 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index fd3a766fc2c..98bb7c1f53d 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -60,7 +60,7 @@ public Field getField() { } public boolean isSet(){ - return !data.getAccessor().isNull(idx()); + return !data.isNull(idx()); } public void read(UnionHolder holder) { diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 3da383fe6f3..e44edbd47b6 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -63,7 +63,7 @@ public class UnionVector implements FieldVector { int valueCount; MapVector internalMap; - ArrowBuf typeBuffer; + protected ArrowBuf typeBuffer; private NullableMapVector mapVector; private ListVector listVector; @@ -121,15 +121,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - typeBuffer.readerIndex(0); - typeBuffer.writerIndex(valueCount * TYPE_WIDTH); - + setReaderAndWriterIndex(); result.add(typeBuffer); return result; } + private void setReaderAndWriterIndex() { + typeBuffer.readerIndex(0); + typeBuffer.writerIndex(valueCount * TYPE_WIDTH); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -232,6 +234,8 @@ public int getTypeValue(int index) { @Override public void allocateNew() throws OutOfMemoryException { + /* new allocation -- clear the current buffers */ + clear(); internalMap.allocateNew(); try { allocateTypeBuffer(); @@ -243,6 +247,8 @@ public void allocateNew() throws OutOfMemoryException { @Override public boolean allocateNewSafe() { + /* new allocation -- clear the current buffers */ + clear(); boolean safe = internalMap.allocateNewSafe(); if (!safe) { return false; } try { @@ -386,6 +392,7 @@ public void transfer() { to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalMapVectorTransferPair.transfer(); to.valueCount = valueCount; + clear(); } @Override @@ -460,8 +467,17 @@ public int getBufferSizeFor(final int valueCount) { @Override public ArrowBuf[] getBuffers(boolean clear) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(typeBuffer); - builder.add(internalMap.getBuffers(clear)); + setReaderAndWriterIndex(); + if (getBufferSize() != 0) { + builder.add(typeBuffer); + builder.add(internalMap.getBuffers(clear)); + } + if (clear) { + valueCount = 0; + typeBuffer.retain(); + typeBuffer.release(); + typeBuffer = allocator.getEmpty(); + } List list = builder.build(); return list.toArray(new ArrowBuf[list.size()]); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index 83dc409a74f..97dfa247845 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -41,7 +41,7 @@ * implying that zero or more elements in the vector could be NULL. */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector - implements FixedWidthVector, FieldVector { + implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter { private final byte typeWidth; protected int valueAllocationSizeInBytes; @@ -314,20 +314,6 @@ public void allocateNew(int valueCount) { valueBufferSize = validityBufferSize; } - if (allocationMonitor > 10) { - /* step down the default memory allocation since we have observed - * multiple times that provisioned value capacity was much larger than - * actually needed. see setValueCount for more details. - */ - valueBufferSize = Math.max(8, valueBufferSize / 2); - validityBufferSize = Math.max(8, validityBufferSize / 2); - allocationMonitor = 0; - } else if (allocationMonitor < -2) { - valueBufferSize = valueBufferSize * 2L; - validityBufferSize = validityBufferSize * 2L; - allocationMonitor = 0; - } - if (valueBufferSize > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } @@ -359,10 +345,9 @@ private void allocateBytes(final long valueBufferSize, final long validityBuffer valueBuffer = allocator.buffer(curSize); valueBuffer.readerIndex(0); valueAllocationSizeInBytes = curSize; - /* allocate validity buffer */ allocateValidityBuffer((int)validityBufferSize); - initValidityBuffer(); + zeroVector(); } /** @@ -375,7 +360,6 @@ private void allocateValidityBuffer(final int validityBufferSize) { validityBuffer = allocator.buffer(validityBufferSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = validityBufferSize; - initValidityBuffer(); } /** @@ -423,9 +407,15 @@ public Field getField() { */ @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = new ArrowBuf[2]; - buffers[0] = validityBuffer; - buffers[1] = valueBuffer; + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[2]; + buffers[0] = validityBuffer; + buffers[1] = valueBuffer; + } if (clear) { for (final ArrowBuf buffer:buffers) { buffer.retain(1); @@ -543,21 +533,28 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers */ public List getFieldBuffers() { List result = new ArrayList<>(2); + setReaderAndWriterIndex(); + result.add(validityBuffer); + result.add(valueBuffer); + return result; + } + + private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); valueBuffer.readerIndex(0); - if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ - valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + valueBuffer.writerIndex(0); } else { - valueBuffer.writerIndex(valueCount * typeWidth); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + if (typeWidth == 0) { + /* specialized handling for NullableBitVector */ + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } else { + valueBuffer.writerIndex(valueCount * typeWidth); + } } - - result.add(validityBuffer); - result.add(valueBuffer); - - return result; } /** @@ -767,6 +764,7 @@ public void setValueCount(int valueCount) { decrementAllocationMonitor(); } } + setReaderAndWriterIndex(); } /** @@ -808,6 +806,7 @@ public int isSet(int index) { * * @param index position of the element. */ + @Override public void setIndexDefined(int index) { handleSafe(index); BitVectorHelper.setValidityBitToOne(validityBuffer, index); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index 47daeee7f20..6996a446b9b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -24,6 +24,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -31,12 +32,13 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; public abstract class BaseNullableVariableWidthVector extends BaseValueVector - implements VariableWidthVector, FieldVector { + implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter { private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; @@ -307,14 +309,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers */ public List getFieldBuffers() { List result = new ArrayList<>(3); - final int lastDataOffset = getstartOffset(valueCount); - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.readerIndex(0); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); - valueBuffer.readerIndex(0); - valueBuffer.writerIndex(lastDataOffset); - + setReaderAndWriterIndex(); result.add(validityBuffer); result.add(offsetBuffer); result.add(valueBuffer); @@ -322,6 +317,22 @@ public List getFieldBuffers() { return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + offsetBuffer.readerIndex(0); + valueBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + offsetBuffer.writerIndex(0); + valueBuffer.writerIndex(0); + } else { + final int lastDataOffset = getstartOffset(valueCount); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + valueBuffer.writerIndex(lastDataOffset); + } + } + /** * Same as {@link #allocateNewSafe()}. */ @@ -430,11 +441,11 @@ private void allocateValidityBuffer(final long size) { * double the current value capacity. */ public void reAlloc() { - reallocValueBuffer(); + reallocDataBuffer(); reallocValidityAndOffsetBuffers(); } - protected void reallocValueBuffer() { + public void reallocDataBuffer() { long baseSize = valueAllocationSizeInBytes; final int currentBufferCapacity = valueBuffer.capacity(); @@ -456,7 +467,7 @@ protected void reallocValueBuffer() { valueAllocationSizeInBytes = (int)newAllocationSize; } - protected void reallocValidityAndOffsetBuffers() { + public void reallocValidityAndOffsetBuffers() { offsetBuffer = reallocBufferHelper(offsetBuffer, true); validityBuffer = reallocBufferHelper(validityBuffer, false); } @@ -560,10 +571,16 @@ public Field getField() { */ @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = new ArrowBuf[3]; - buffers[0] = validityBuffer; - buffers[1] = offsetBuffer; - buffers[1] = valueBuffer; + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[3]; + buffers[0] = validityBuffer; + buffers[1] = offsetBuffer; + buffers[2] = valueBuffer; + } if (clear) { for (final ArrowBuf buffer:buffers) { buffer.retain(1); @@ -615,8 +632,10 @@ public void transferTo(BaseNullableVariableWidthVector target){ target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; - target.valueCount = valueCount; - target.setLastSet(lastSet); + target.setLastSet(this.lastSet); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } clear(); } @@ -634,7 +653,9 @@ public void splitAndTransferTo(int startIndex, int length, splitAndTransferValidityBuffer(startIndex, length, target); splitAndTransferOffsetBuffer(startIndex, length, target); target.setLastSet(length - 1); - target.setValueCount(length); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } } /* @@ -793,6 +814,7 @@ public void setValueCount(int valueCount) { } fillHoles(valueCount); lastSet = valueCount - 1; + setReaderAndWriterIndex(); } /** @@ -836,7 +858,7 @@ public int getLastSet() { * @return starting offset for the element */ public long getStartEnd(int index) { - return (long)offsetBuffer.getInt(index * OFFSET_WIDTH); + return offsetBuffer.getLong(index * OFFSET_WIDTH); } /** @@ -844,9 +866,251 @@ public long getStartEnd(int index) { * * @param index position of the element. */ + @Override public void setIndexDefined(int index) { - handleSafe(index, 0); + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + fillHoles(index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + lastSet = index; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if(isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index){ + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillEmpties(index); + handleSafe(index, end); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; } @@ -905,7 +1169,7 @@ protected final void handleSafe(int index, int dataLength) { } final int startOffset = getstartOffset(index); while (valueBuffer.capacity() < (startOffset + dataLength)) { - reallocValueBuffer(); + reallocDataBuffer(); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 591d13c4bd5..c6d404e1543 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -251,12 +251,12 @@ public void copyFromSafe(int inIndex, int outIndex, BitVector from) { @Override public Mutator getMutator() { - return new Mutator(); + return mutator; } @Override public Accessor getAccessor() { - return new Accessor(); + return accessor; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java new file mode 100644 index 00000000000..78304b354cd --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -0,0 +1,337 @@ +/******************************************************************************* + + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package org.apache.arrow.vector; + +import java.math.BigDecimal; +import java.nio.charset.Charset; + +/** + * Helper class to generate test data for Nullable fixed and variable + * width scalar vectors. Previous implementations of java vector classes + * provided generateTestData(now deprecated) API to populate the vector + * with sample data. This class should be used for that purpose. + */ +public class GenerateSampleData { + + public static void generateTestData(final ValueVector vector, final int valueCount) { + if (vector instanceof NullableIntVector) { + writeIntData((NullableIntVector)vector, valueCount); + } else if (vector instanceof NullableDecimalVector) { + writeDecimalData((NullableDecimalVector)vector, valueCount); + } else if (vector instanceof NullableBitVector) { + writeBooleanData((NullableBitVector)vector, valueCount); + } else if (vector instanceof NullableVarCharVector) { + writeVarCharData((NullableVarCharVector)vector, valueCount); + } else if (vector instanceof NullableVarBinaryVector) { + writeVarBinaryData((NullableVarBinaryVector)vector, valueCount); + } else if (vector instanceof NullableBigIntVector) { + writeBigIntData((NullableBigIntVector) vector, valueCount); + } else if (vector instanceof NullableFloat4Vector) { + writeFloatData((NullableFloat4Vector)vector, valueCount); + } else if (vector instanceof NullableFloat8Vector) { + writeDoubleData((NullableFloat8Vector) vector, valueCount); + } else if (vector instanceof NullableDateDayVector) { + writeDateDayData((NullableDateDayVector)vector, valueCount); + } else if (vector instanceof NullableDateMilliVector) { + writeDateMilliData((NullableDateMilliVector) vector, valueCount); + } else if (vector instanceof NullableIntervalDayVector) { + writeIntervalDayData((NullableIntervalDayVector)vector, valueCount); + } else if (vector instanceof NullableIntervalYearVector) { + writeIntervalYearData((NullableIntervalYearVector) vector, valueCount); + } else if (vector instanceof NullableSmallIntVector) { + writeSmallIntData((NullableSmallIntVector)vector, valueCount); + } else if (vector instanceof NullableTinyIntVector) { + writeTinyIntData((NullableTinyIntVector) vector, valueCount); + } else if (vector instanceof NullableTimeMicroVector) { + writeTimeMicroData((NullableTimeMicroVector)vector, valueCount); + } else if (vector instanceof NullableTimeMilliVector) { + writeTimeMilliData((NullableTimeMilliVector) vector, valueCount); + } else if (vector instanceof NullableTimeNanoVector) { + writeTimeNanoData((NullableTimeNanoVector)vector, valueCount); + } else if (vector instanceof NullableTimeSecVector) { + writeTimeSecData((NullableTimeSecVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecVector) { + writeTimeStampData((NullableTimeStampSecVector)vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroVector) { + writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliVector) { + writeTimeStampData((NullableTimeStampMilliVector)vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoVector) { + writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecTZVector) { + writeTimeStampData((NullableTimeStampSecTZVector)vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroTZVector) { + writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliTZVector) { + writeTimeStampData((NullableTimeStampMilliTZVector)vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoTZVector) { + writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount); + } + } + + private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) { + final long even = 100000; + final long odd = 200000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDecimalData(NullableDecimalVector vector, int valueCount) { + final BigDecimal even = new BigDecimal(0.0543278923); + final BigDecimal odd = new BigDecimal(2.0543278923); + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntData(NullableIntVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeBooleanData(NullableBitVector vector, int valueCount) { + final int even = 0; + final int odd = 1; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) { + final int even = 1; + final int odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) { + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, 1, 50); + } else { + vector.setSafe(i, 2, 100); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) { + final int even = 500; + final int odd = 900; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + + } + + private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDateDayData(NullableDateDayVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) { + final short even = 10; + final short odd = 20; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) { + final byte even = 1; + final byte odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeBigIntData(NullableBigIntVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeFloatData(NullableFloat4Vector vector, int valueCount) { + final float even = 20.3f; + final float odd = 40.2f; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) { + final double even = 20.2373; + final double odd = 40.2378; + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } + + private static void writeVarCharData(NullableVarCharVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i%2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); + } + } + vector.setValueCount(valueCount); + } +} + diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index 253427333a2..ae896fb7907 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -125,7 +125,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } @@ -139,6 +139,8 @@ public Long getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java index eceaa61b57f..dda4c434876 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -163,7 +163,7 @@ private ArrowBuf splitAndTransferBuffer(int startIndex, int length, if (destBuffer != null) { destBuffer.release(); } - destBuffer = destBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); destBuffer.retain(1); } else { @@ -279,6 +279,8 @@ public Boolean getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index e6b5b590f3f..566cb2729d4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -127,7 +127,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getInt(index * TYPE_WIDTH); } } @@ -141,6 +141,8 @@ public Integer getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index 8e15100f708..4d63be1abcd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -129,7 +129,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = get(index); + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return localDateTime; } @@ -145,6 +145,8 @@ public LocalDateTime getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 2a611c6a2d5..9d3b2a7787a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -154,6 +154,9 @@ public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { if (from.isSet(fromIndex) != 0) { from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, thisIndex * TYPE_WIDTH, TYPE_WIDTH); + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index 3ba5cfcea52..eb895a29beb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -126,7 +126,7 @@ public Float getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getFloat(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Float getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index 2fb96a44fe7..51aac5ffead 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -126,7 +126,7 @@ public Double getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getDouble(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Double getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 93deacbdfdc..1fadbf23dce 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -126,7 +126,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getInt(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Integer getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java index 174b722b043..8524905cf37 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -190,6 +190,8 @@ public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector fro BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java index 303286d0ed3..6aeae899289 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -126,7 +126,7 @@ public Period getObject(int index) { if (isSet(index) == 0) { return null; } else { - final int interval = get(index); + final int interval = valueBuffer.getInt(index * TYPE_WIDTH); final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); final Period p = new Period(); @@ -172,6 +172,8 @@ private StringBuilder getAsStringBuilderHelper(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index ed337188b5b..ca57b03c032 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -126,7 +126,7 @@ public Short getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getShort(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Short getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index d7ea3222397..92d8bec0864 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -127,7 +127,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } @@ -141,6 +141,8 @@ public Long getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 07d8abba4e5..5b7bd39f99b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -142,6 +142,8 @@ public LocalDateTime getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index 947b2392f7a..100e4fd0711 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -126,7 +126,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Long getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index 0a3cfaac047..de512db3591 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -126,7 +126,7 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getInt(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Integer getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java index 0d25eff4e02..6b53deb6b6b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -115,7 +115,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java index 41f7a1112f2..7380ff5df72 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -113,7 +113,7 @@ public LocalDateTime getObject(int index) { return null; } else { /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ - final long micros = get(index); + final long micros = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return localDateTime; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java index 6376b085e64..2bb7c003a0a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -115,7 +115,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java index ff581d2ea2f..6550c405e38 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -112,7 +112,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long millis = get(index); + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return localDateTime; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java index 6c4445d500b..47fb033aae6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -115,7 +115,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java index 096913ebf1c..affc4780057 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -112,7 +112,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long nanos = get(index); + final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return localDateTime; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java index 8e5ba72ae1f..3096f8c558b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -114,7 +114,7 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java index bc800e0e4f0..6690e5a2c24 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -112,7 +112,7 @@ public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; } else { - final long secs = get(index); + final long secs = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return localDateTime; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index 1bf2abc384d..9a8771ae7bb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -72,6 +72,8 @@ public long get(int index) throws IllegalStateException { public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index ccbfa32b85f..578c1e18594 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -126,7 +126,7 @@ public Byte getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getByte(index * TYPE_WIDTH); } } @@ -140,6 +140,8 @@ public Byte getObject(int index) { public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java index 4d20f800ba1..ba88b96c7c2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -103,13 +103,15 @@ public Byte getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getByte(index * TYPE_WIDTH); } } public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java index b140b9f5cbc..9a097a60c48 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -104,13 +104,15 @@ public Character getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getChar(index * TYPE_WIDTH); } } public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java index fc8cb639581..0527b10e1ea 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -104,13 +104,15 @@ public Integer getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getInt(index * TYPE_WIDTH); } } public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java index 2110cf03b9f..4f28003e9f1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -104,13 +104,15 @@ public Long getObject(int index) { if (isSet(index) == 0) { return null; } else { - return get(index); + return valueBuffer.getLong(index * TYPE_WIDTH); } } public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { if (from.isSet(fromIndex) != 0) { set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java index 330e06710fc..7a81733833e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -122,23 +122,6 @@ public byte[] getObject(int index) { return b; } - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return greater than 0 length for non-null element, 0 otherwise - */ - public int getValueLength(int index) { - assert index >= 0; - if(isSet(index) == 0) { - return 0; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - return dataLength; - } - /** * Get the variable length element at specified index and sets the state * in provided holder. @@ -152,12 +135,9 @@ public void get(int index, NullableVarBinaryHolder holder){ holder.isSet = 0; return; } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; holder.isSet = 1; - holder.start = startOffset; - holder.end = dataLength; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); holder.buffer = valueBuffer; } @@ -178,11 +158,15 @@ public void get(int index, NullableVarBinaryHolder holder){ * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; fillHoles(thisIndex); - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; } /** @@ -194,121 +178,16 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) * @param from source vector */ public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { - fillEmpties(thisIndex); - if (from.isSet(fromIndex) != 0) { - setSafe(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - - /** - * Set the variable length element at the specified index to the supplied - * byte array. This is same as using {@link #set(int, byte[], int, int)} - * with start as 0 and length as value.length - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void set(int index, byte[] value) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[])} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void setSafe(int index, byte[] value) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, value.length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void set(int index, byte[] value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[], int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void setSafe(int index, byte[] value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the - * content in supplied ByteBuffer - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void set(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void setSafe(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; } /** @@ -387,70 +266,6 @@ public void setSafe(int index, NullableVarBinaryHolder holder) { lastSet = index; } - /** - * Sets the value length for an element. - * - * @param index position of the element to set - * @param length length of the element - */ - public void setValueLengthSafe(int index, int length) { - assert index >= 0; - handleSafe(index, length); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index, 0); - /* not really needed to set the bit to 0 as long as - * the buffer always starts from 0. - */ - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); - final ArrowBuf bb = buffer.slice(start, end); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - handleSafe(index, end); - set(index, isSet, start, end, buffer); - } - /****************************************************************** * * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index 0b03164f0f3..c1ebf4b7365 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -125,23 +125,6 @@ public Text getObject(int index) { return result; } - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return greater than 0 length for non-null element, 0 otherwise - */ - public int getValueLength(int index) { - assert index >= 0; - if(isSet(index) == 0) { - return 0; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - return dataLength; - } - /** * Get the variable length element at specified index and sets the state * in provided holder. @@ -155,12 +138,9 @@ public void get(int index, NullableVarCharHolder holder){ holder.isSet = 0; return; } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; holder.isSet = 1; - holder.start = startOffset; - holder.end = dataLength; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); holder.buffer = valueBuffer; } @@ -181,11 +161,15 @@ public void get(int index, NullableVarCharHolder holder){ * @param from source vector */ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; fillHoles(thisIndex); - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; } /** @@ -197,121 +181,16 @@ public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { * @param from source vector */ public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { - fillEmpties(thisIndex); - if (from.isSet(fromIndex) != 0) { - setSafe(thisIndex, from.get(fromIndex)); - lastSet = thisIndex; - } - } - - - /** - * Set the variable length element at the specified index to the supplied - * byte array. This is same as using {@link #set(int, byte[], int, int)} - * with start as 0 and length as value.length - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void set(int index, byte[] value) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[])} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void setSafe(int index, byte[] value) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, value.length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void set(int index, byte[] value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[], int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void setSafe(int index, byte[] value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the - * content in supplied ByteBuffer - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void set(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void setSafe(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; } /** @@ -390,70 +269,6 @@ public void setSafe(int index, NullableVarCharHolder holder) { lastSet = index; } - /** - * Sets the value length for an element. - * - * @param index position of the element to set - * @param length length of the element - */ - public void setValueLengthSafe(int index, int length) { - assert index >= 0; - handleSafe(index, length); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index, 0); - /* not really needed to set the bit to 0 as long as - * the buffer always starts from 0. - */ - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + end); - final ArrowBuf bb = buffer.slice(start, end); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - handleSafe(index, end); - set(index, isSet, start, end, buffer); - } - /****************************************************************** * * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 3eab30a32d2..f09d855be58 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -147,7 +147,7 @@ public int getValueCapacity() { return Math.min(vector.getValueCapacity(), offsetValueCapacity); } - private int getOffsetBufferValueCapacity() { + protected int getOffsetBufferValueCapacity() { return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); } @@ -183,8 +183,12 @@ public void clear() { @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, - vector.getBuffers(false), ArrowBuf.class); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index bb091f68c18..1312495d5e9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -134,15 +134,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - + setReaderAndWriterIndex(); result.add(validityBuffer); return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -182,6 +184,8 @@ public boolean allocateNewSafe() { */ boolean success = false; try { + /* we are doing a new allocation -- release the current buffers */ + clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); success = vector.allocateNewSafe(); @@ -282,7 +286,13 @@ public void clear() { @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -462,6 +472,7 @@ public void transfer() { dataPair.transfer(); to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.setValueCount(valueCount); + clear(); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index ea577208cc2..22b631204ab 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -60,7 +60,7 @@ public static ListVector empty(String name, BufferAllocator allocator) { return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); } - private ArrowBuf validityBuffer; + protected ArrowBuf validityBuffer; private UnionListReader reader; private CallBack callBack; private final FieldType fieldType; @@ -132,17 +132,25 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(2); - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.readerIndex(0); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); - + setReaderAndWriterIndex(); result.add(validityBuffer); result.add(offsetBuffer); return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + offsetBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + offsetBuffer.writerIndex(0); + } else { + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + } + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -163,6 +171,8 @@ public void allocateNew() throws OutOfMemoryException { public boolean allocateNewSafe() { boolean success = false; try { + /* we are doing a new allocation -- release the current buffers */ + clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); /* allocate offset and data buffer */ @@ -299,11 +309,15 @@ public TransferImpl(ListVector to) { @Override public void transfer() { + to.clear(); dataTransferPair.transfer(); to.validityBuffer = validityBuffer.transferOwnership(to.allocator).buffer; to.offsetBuffer = offsetBuffer.transferOwnership(to.allocator).buffer; to.lastSet = lastSet; - to.setValueCount(valueCount); + if (valueCount > 0) { + to.setValueCount(valueCount); + } + clear(); } @Override @@ -410,7 +424,7 @@ public AddOrGetResult addOrGetVector(FieldType fieldT @Override public int getBufferSize() { - if (getValueCount() == 0) { + if (valueCount == 0) { return 0; } final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; @@ -437,8 +451,15 @@ public void clear() { @Override public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, - vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, + ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, + vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); + } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); @@ -492,8 +513,22 @@ public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } + @Override + public int getValueCapacity() { + return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); + } + + public int getValidityAndOffsetValueCapacity() { + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + private int getValidityBufferValueCapacity() { + return (int)(validityBuffer.capacity() * 8L); + } + public void setNotNull(int index) { - if (index >= getValueCapacity()) { + while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); } BitVectorHelper.setValidityBitToOne(validityBuffer, index); @@ -502,14 +537,14 @@ public void setNotNull(int index) { @Override public int startNewValue(int index) { - if (index >= getValueCapacity()) { + while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); } for (int i = lastSet; i <= index; i++) { final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH); offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset); } - setNotNull(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); lastSet = index + 1; return offsetBuffer.getInt(lastSet * OFFSET_WIDTH); } @@ -529,8 +564,8 @@ public void endValue(int index, int size) { public void setValueCount(int valueCount) { this.valueCount = valueCount; if (valueCount > 0) { - while (valueCount > getValueCapacity()) { - /* realloc the inner buffers if needed */ + while (valueCount > getValidityAndOffsetValueCapacity()) { + /* check if validity and offset buffers need to be re-allocated */ reallocValidityAndOffsetBuffers(); } for (int i = lastSet; i < valueCount; i++) { @@ -542,6 +577,9 @@ public void setValueCount(int valueCount) { /* valueCount for the data vector is the current end offset */ final int childValueCount = (valueCount == 0) ? 0 : offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + /* set the value count of data vector and this will take care of + * checking whether data buffer needs to be reallocated. + */ vector.setValueCount(childValueCount); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index a1478c5a721..724f819fe04 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -54,7 +54,7 @@ public static NullableMapVector empty(String name, BufferAllocator allocator) { private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); - private ArrowBuf validityBuffer; + protected ArrowBuf validityBuffer; private int validityAllocationSizeInBytes; // deprecated, use FieldType or static constructor instead @@ -99,14 +99,17 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers @Override public List getFieldBuffers() { List result = new ArrayList<>(1); - - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + setReaderAndWriterIndex(); result.add(validityBuffer); return result; } + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + } + @Override @Deprecated public List getFieldInnerVectors() { @@ -156,6 +159,7 @@ public void transfer() { target.clear(); target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; super.transfer(); + clear(); } @Override @@ -248,10 +252,21 @@ public int getValueCapacity() { @Override public ArrowBuf[] getBuffers(boolean clear) { + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), ArrowBuf.class); + } if (clear) { - validityBuffer.retain(1); + for (ArrowBuf buffer: buffers) { + buffer.retain(); + } + clear(); } - return ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(clear), ArrowBuf.class); + + return buffers; } @Override @@ -302,7 +317,7 @@ public boolean allocateNewSafe() { */ boolean success = false; try { - clearValidityBuffer(); + clear(); allocateValidityBuffer(validityAllocationSizeInBytes); success = super.allocateNewSafe(); } finally { @@ -431,7 +446,7 @@ public void setNull(int index) { @Override public void setValueCount(int valueCount) { assert valueCount >= 0; - while (valueCount > getValueCapacity()) { + while (valueCount > getValidityBufferValueCapacity()) { /* realloc the inner buffers if needed */ reallocValidityBuffer(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index ede4f992291..aec7d0f3273 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import java.util.List; @@ -113,10 +114,23 @@ public void testTransfer() throws Exception { assertEquals(srcVector.getField(), destVector.getField()); // now check the values are transferred - assertEquals(srcVector.getValueCount(), destVector.getValueCount()); - for (int i = 0; i < srcVector.getValueCount(); i++) { - assertEquals("Different values at index " + i, srcVector.get(i), destVector.get(i)); - } + assertEquals(6, destVector.getValueCount()); + + assertFalse(destVector.isNull(0)); + assertEquals(5, destVector.getObject(0)); + + assertFalse(destVector.isNull(1)); + assertEquals(false, destVector.getObject(1)); + + assertTrue(destVector.isNull(2)); + + assertFalse(destVector.isNull(3)); + assertEquals(10, destVector.getObject(3)); + + assertTrue(destVector.isNull(4)); + + assertFalse(destVector.isNull(5)); + assertEquals(false, destVector.getObject(5)); } } } From 102ff9727a62a0eb411c283f173f7a329fb056be Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 13 Nov 2017 20:44:15 -0800 Subject: [PATCH 05/10] indentation and line length --- .../java/org/apache/arrow/vector/NullableDateMilliVector.java | 3 ++- .../org/apache/arrow/vector/NullableTimeStampMicroVector.java | 3 ++- .../org/apache/arrow/vector/NullableTimeStampMilliVector.java | 3 ++- .../org/apache/arrow/vector/NullableTimeStampNanoVector.java | 3 ++- .../org/apache/arrow/vector/NullableTimeStampSecVector.java | 3 ++- .../apache/arrow/vector/complex/BaseRepeatedValueVector.java | 3 ++- .../org/apache/arrow/vector/complex/FixedSizeListVector.java | 3 ++- .../org/apache/arrow/vector/complex/NullableMapVector.java | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index 4d63be1abcd..0f439fde746 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -130,7 +130,8 @@ public LocalDateTime getObject(int index) { return null; } else { final long millis = valueBuffer.getLong(index * TYPE_WIDTH); - final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); return localDateTime; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java index 7380ff5df72..9debe1daad3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -115,7 +115,8 @@ public LocalDateTime getObject(int index) { /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ final long micros = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); return localDateTime; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java index 6550c405e38..8f4bf4acb99 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -113,7 +113,8 @@ public LocalDateTime getObject(int index) { return null; } else { final long millis = valueBuffer.getLong(index * TYPE_WIDTH); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); return localDateTime; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java index affc4780057..d464873e8d2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -114,7 +114,8 @@ public LocalDateTime getObject(int index) { } else { final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); return localDateTime; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java index 6690e5a2c24..1c281ef492f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -114,7 +114,8 @@ public LocalDateTime getObject(int index) { } else { final long secs = valueBuffer.getLong(index * TYPE_WIDTH); final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); return localDateTime; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index f09d855be58..aab5aabd65c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -187,7 +187,8 @@ public ArrowBuf[] getBuffers(boolean clear) { if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), ArrowBuf.class); + buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 1312495d5e9..f017830c8ce 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -291,7 +291,8 @@ public ArrowBuf[] getBuffers(boolean clear) { if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class); + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 724f819fe04..1c0e5df4de6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -257,7 +257,8 @@ public ArrowBuf[] getBuffers(boolean clear) { if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), ArrowBuf.class); + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer: buffers) { From 343f47bd5b55db92967ab6d1fa3148453a9bb2a2 Mon Sep 17 00:00:00 2001 From: siddharth Date: Mon, 13 Nov 2017 22:43:47 -0800 Subject: [PATCH 06/10] rebase on master, fix tests --- .../arrow/vector/NullableDecimalVector.java | 5 +++++ .../arrow/vector/file/json/JsonFileReader.java | 8 ++++---- .../arrow/vector/file/json/JsonFileWriter.java | 16 ++++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 9d3b2a7787a..20ee3800955 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -173,6 +173,11 @@ public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector fro copyFrom(fromIndex, thisIndex, from); } + /** + * Return scale for the decimal value + */ + public int getScale() { return scale; } + /****************************************************************** * * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index d3c8b990394..61a591d81fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -27,7 +27,8 @@ import java.io.File; import java.io.IOException; - +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -313,11 +314,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH); - for (int i = 0; i < count; i++) { parser.nextToken(); - final byte[] value = decodeHexSafe(parser.getValueAsString()); - DecimalUtility.writeByteArrayToArrowBuf(value, buf, i); + BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); + DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i); } return buf; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java index 7921cd34c77..0c8507b51f5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -159,6 +160,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE generator.writeObjectField("name", field.getName()); int valueCount = vector.getValueCount(); generator.writeObjectField("count", valueCount); + final int scale = (vector instanceof NullableDecimalVector) ? + ((NullableDecimalVector) vector).getScale() : 0; for (int v = 0; v < vectorTypes.size(); v++) { ArrowVectorType vectorType = vectorTypes.get(v); ArrowBuf vectorBuffer = vectorBuffers.get(v); @@ -167,9 +170,9 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE for (int i = 0; i < bufferValueCount; i++) { if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || vector.getMinorType() == Types.MinorType.VARBINARY)) { - writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i); + writeValueToGenerator(vectorType, vectorBuffer, vectorBuffers.get(v-1), vector, i, scale); } else { - writeValueToGenerator(vectorType, vectorBuffer, null, vector, i); + writeValueToGenerator(vectorType, vectorBuffer, null, vector, i, scale); } } generator.writeEndArray(); @@ -193,7 +196,8 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE } private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, - ArrowBuf offsetBuffer, FieldVector vector, int index) throws IOException { + ArrowBuf offsetBuffer, FieldVector vector, + final int index, final int scale) throws IOException { if (bufferType.equals(TYPE)) { generator.writeNumber(buffer.getByte(index * NullableTinyIntVector.TYPE_WIDTH)); } else if (bufferType.equals(OFFSET)) { @@ -279,9 +283,9 @@ private void writeValueToGenerator(ArrowVectorType bufferType, ArrowBuf buffer, break; } case DECIMAL: { - String hexString = Hex.encodeHexString(DecimalUtility.getByteArrayFromArrowBuf(buffer, - index)); - generator.writeString(hexString); + BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale); + // We write the unscaled value, because the scale is stored in the type metadata. + generator.writeString(decimalValue.unscaledValue().toString()); break; } default: From 635ca77028bbf86f549a9598ba5d61f0e32f9573 Mon Sep 17 00:00:00 2001 From: siddharth Date: Tue, 14 Nov 2017 11:18:59 -0800 Subject: [PATCH 07/10] address review comments --- .../BaseNullableVariableWidthVector.java | 31 +++++++++++++++++++ .../arrow/vector/GenerateSampleData.java | 2 +- .../complex/BaseRepeatedValueVector.java | 2 +- .../vector/complex/FixedSizeListVector.java | 2 +- .../arrow/vector/complex/ListVector.java | 2 +- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index 6996a446b9b..e08f6e5961e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -445,6 +445,14 @@ public void reAlloc() { reallocValidityAndOffsetBuffers(); } + /** + * Reallocate the data buffer. Data Buffer stores the actual data for + * VARCHAR or VARBINARY elements in the vector. The behavior is to double + * the size of buffer. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ public void reallocDataBuffer() { long baseSize = valueAllocationSizeInBytes; final int currentBufferCapacity = valueBuffer.capacity(); @@ -467,6 +475,29 @@ public void reallocDataBuffer() { valueAllocationSizeInBytes = (int)newAllocationSize; } + /** + * Reallocate the validity and offset buffers for this vector. Validity + * buffer is used to track the NULL or NON-NULL nature of elements in + * the vector and offset buffer is used to store the lengths of variable + * width elements in the vector. + * + * Note that data buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ public void reallocValidityAndOffsetBuffers() { offsetBuffer = reallocBufferHelper(offsetBuffer, true); validityBuffer = reallocBufferHelper(validityBuffer, false); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java index 78304b354cd..77d214bded6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -1,5 +1,5 @@ /******************************************************************************* - + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index aab5aabd65c..1489542b0fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -188,7 +188,7 @@ public ArrowBuf[] getBuffers(boolean clear) { buffers = new ArrowBuf[0]; } else { buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), - ArrowBuf.class); + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index f017830c8ce..4c3b4b2a86c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -292,7 +292,7 @@ public ArrowBuf[] getBuffers(boolean clear) { buffers = new ArrowBuf[0]; } else { buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), - ArrowBuf.class); + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 22b631204ab..1134ac88cd5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -457,7 +457,7 @@ public ArrowBuf[] getBuffers(boolean clear) { buffers = new ArrowBuf[0]; } else { buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, - ObjectArrays.concat(new ArrowBuf[] {validityBuffer}, + ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class); } if (clear) { From 97061724db140eafe5d7168796ece054805edd53 Mon Sep 17 00:00:00 2001 From: siddharth Date: Tue, 14 Nov 2017 19:15:17 -0800 Subject: [PATCH 08/10] integration test fix, some checkstyle fixes --- .../codegen/templates/LegacyUnionVector.java | 356 --- .../templates/NullableValueVectors.java | 595 ----- .../arrow/vector/BaseDataValueVector.java | 6 +- .../vector/BaseNullableFixedWidthVector.java | 1439 +++++------ .../BaseNullableVariableWidthVector.java | 2253 ++++++++--------- .../apache/arrow/vector/BaseValueVector.java | 4 +- .../apache/arrow/vector/BitVectorHelper.java | 288 ++- .../arrow/vector/GenerateSampleData.java | 5 +- .../arrow/vector/NullableBigIntVector.java | 654 +++-- .../arrow/vector/NullableBitVector.java | 890 +++---- .../arrow/vector/NullableDateDayVector.java | 656 +++-- .../arrow/vector/NullableDateMilliVector.java | 664 +++-- .../arrow/vector/NullableDecimalVector.java | 766 +++--- .../arrow/vector/NullableFloat4Vector.java | 656 +++-- .../arrow/vector/NullableFloat8Vector.java | 656 +++-- .../arrow/vector/NullableIntVector.java | 666 ++--- .../vector/NullableIntervalDayVector.java | 774 +++--- .../vector/NullableIntervalYearVector.java | 694 +++-- .../arrow/vector/NullableSmallIntVector.java | 710 +++--- .../arrow/vector/NullableTimeMicroVector.java | 654 +++-- .../arrow/vector/NullableTimeMilliVector.java | 658 +++-- .../arrow/vector/NullableTimeNanoVector.java | 654 +++-- .../arrow/vector/NullableTimeSecVector.java | 656 +++-- .../NullableTimeStampMicroTZVector.java | 362 ++- .../vector/NullableTimeStampMicroVector.java | 364 ++- .../NullableTimeStampMilliTZVector.java | 360 ++- .../vector/NullableTimeStampMilliVector.java | 362 ++- .../vector/NullableTimeStampNanoTZVector.java | 362 ++- .../vector/NullableTimeStampNanoVector.java | 364 ++- .../vector/NullableTimeStampSecTZVector.java | 360 ++- .../vector/NullableTimeStampSecVector.java | 364 ++- .../arrow/vector/NullableTimeStampVector.java | 370 +-- .../arrow/vector/NullableTinyIntVector.java | 710 +++--- .../arrow/vector/NullableUInt1Vector.java | 556 ++-- .../arrow/vector/NullableUInt2Vector.java | 556 ++-- .../arrow/vector/NullableUInt4Vector.java | 500 ++-- .../arrow/vector/NullableUInt8Vector.java | 500 ++-- .../arrow/vector/NullableVarBinaryVector.java | 583 +++-- .../arrow/vector/NullableVarCharVector.java | 587 +++-- .../org/apache/arrow/vector/ValueVector.java | 4 +- .../complex/BaseRepeatedValueVector.java | 27 +- .../vector/complex/FixedSizeListVector.java | 27 +- .../LegacyBaseRepeatedValueVector.java | 240 -- .../complex/LegacyFixedSizeListVector.java | 298 --- .../vector/complex/LegacyListVector.java | 288 --- .../arrow/vector/complex/LegacyMapVector.java | 229 -- .../arrow/vector/complex/ListVector.java | 43 +- .../vector/complex/NullableMapVector.java | 34 +- .../vector/file/json/JsonFileReader.java | 60 +- 49 files changed, 10940 insertions(+), 12924 deletions(-) delete mode 100644 java/vector/src/main/codegen/templates/LegacyUnionVector.java delete mode 100644 java/vector/src/main/codegen/templates/NullableValueVectors.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java diff --git a/java/vector/src/main/codegen/templates/LegacyUnionVector.java b/java/vector/src/main/codegen/templates/LegacyUnionVector.java deleted file mode 100644 index b9cb6dfdc24..00000000000 --- a/java/vector/src/main/codegen/templates/LegacyUnionVector.java +++ /dev/null @@ -1,356 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -<@pp.dropOutputFile /> -<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/LegacyUnionVector.java" /> - - -<#include "/@includes/license.ftl" /> - - package org.apache.arrow.vector.complex; - -<#include "/@includes/vv_imports.ftl" /> - import com.google.common.collect.ImmutableList; - import java.util.ArrayList; - import java.util.Collections; - import java.util.Iterator; - import org.apache.arrow.vector.BaseDataValueVector; - import org.apache.arrow.vector.complex.impl.ComplexCopier; - import org.apache.arrow.vector.util.CallBack; - import org.apache.arrow.vector.schema.ArrowFieldNode; - - import static org.apache.arrow.vector.types.UnionMode.Sparse; - - - -/* - * This class is generated using freemarker and the ${.template_name} template. - */ -@SuppressWarnings("unused") - - -/** - * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each - * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, - * as well as the addOrGet method. - * - * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup - * each time the vector is accessed. - * Source code generated using FreeMarker template ${.template_name} - */ -public class LegacyUnionVector implements FieldVector { - - private Accessor accessor = new Accessor(); - private Mutator mutator = new Mutator(); - private final UnionVector unionVector; - - public LegacyUnionVector(String name, BufferAllocator allocator, CallBack callBack) { - unionVector = new UnionVector(name, allocator, callBack); - } - - public BufferAllocator getAllocator() { - return unionVector.getAllocator(); - } - - @Override - public MinorType getMinorType() { - return MinorType.UNION; - } - - @Override - public void initializeChildrenFromFields(List children) { - unionVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return unionVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - unionVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return unionVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return unionVector.getFieldInnerVectors(); - } - - @Override - public long getValidityBufferAddress() { - return unionVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return unionVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { throw new UnsupportedOperationException(); } - - @Override - public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - - public NullableMapVector getMap() { - return unionVector.getMap(); - } - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#assign lowerCaseName = name?lower_case/> - <#if !minor.typeParams?? > - - private Nullable${name}Vector ${uncappedName}Vector; - - public Nullable${name}Vector get${name}Vector() { - return unionVector.get${name}Vector(); - } - - - - - public ListVector getList() { - return unionVector.getList(); - } - - public int getTypeValue(int index) { - return unionVector.getTypeValue(index); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - unionVector.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - return unionVector.allocateNewSafe(); - } - - @Override - public void reAlloc() { - unionVector.reAlloc(); - } - - @Override - public void setInitialCapacity(int numRecords) { - } - - @Override - public int getValueCapacity() { - return unionVector.getValueCapacity(); - } - - @Override - public void close() { - unionVector.close(); - } - - @Override - public void clear() { - unionVector.clear(); - } - - @Override - public Field getField() { - return unionVector.getField(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return unionVector.getTransferPair(allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return unionVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return unionVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return unionVector.makeTransferPair(((LegacyUnionVector)target).unionVector); - } - - public void copyFrom(int inIndex, int outIndex, UnionVector from) { - unionVector.copyFrom(inIndex, outIndex, from); - } - - public void copyFromSafe(int inIndex, int outIndex, UnionVector from) { - unionVector.copyFromSafe(inIndex, outIndex, from); - } - - public FieldVector addVector(FieldVector v) { - return unionVector.addVector(v); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public FieldReader getReader() { - return unionVector.getReader(); - } - - public FieldWriter getWriter() { - return unionVector.getWriter(); - } - - @Override - public int getBufferSize() { - return unionVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - return unionVector.getBufferSizeFor(valueCount); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return unionVector.getBuffers(clear); - } - - @Override - public Iterator iterator() { - return unionVector.iterator(); - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return unionVector.getObject(index); - } - - public byte[] get(int index) { - return unionVector.get(index); - } - - public void get(int index, ComplexHolder holder) { - } - - public void get(int index, UnionHolder holder) { - unionVector.get(index, holder); - } - - public int getNullCount() { - return unionVector.getNullCount(); - } - - @Override - public int getValueCount() { - return unionVector.getValueCount(); - } - - @Override - public boolean isNull(int index) { - return unionVector.isNull(index); - } - - public int isSet(int index) { - return unionVector.isSet(index); - } - } - - public class Mutator extends BaseValueVector.BaseMutator { - - UnionWriter writer; - - @Override - public void setValueCount(int valueCount) { - unionVector.setValueCount(valueCount); - } - - public void setSafe(int index, UnionHolder holder) { - unionVector.setSafe(index, holder); - } - <#list vv.types as type> - <#list type.minor as minor> - <#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? > - public void setSafe(int index, Nullable${name}Holder holder) { - unionVector.setSafe(index, holder); - } - - - - - - public void setType(int index, MinorType type) { - unionVector.setType(index, type); - } - - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - @Override - @Deprecated - public int getValueCount() { return getAccessor().getValueCount(); } - - @Override - @Deprecated - public void setValueCount(int valueCount) { getMutator().setValueCount(valueCount);} - - @Override - @Deprecated - public Object getObject(int index) { return getAccessor().getObject(index); } - - @Override - @Deprecated - public int getNullCount() { return getAccessor().getNullCount(); } - - @Override - @Deprecated - public boolean isNull(int index) { return getAccessor().isNull(index); } -} diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java deleted file mode 100644 index 8e5b10b649b..00000000000 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ /dev/null @@ -1,595 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -<@pp.dropOutputFile /> -<#list vv.types as type> -<#list type.minor as minor> - -<#assign className = "LegacyNullable${minor.class}Vector" /> -<#assign valuesName = "Nullable${minor.class}Vector" /> -<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> - -<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> - -<#include "/@includes/license.ftl" /> - -package org.apache.arrow.vector; - -import org.apache.arrow.vector.schema.ArrowFieldNode; -import java.util.Collections; - -<#include "/@includes/vv_imports.ftl" /> - -import org.apache.arrow.flatbuf.Precision; - -/** - * ${className} implements a vector of values which could be null. Elements in the vector - * are first checked against a fixed length vector of boolean values. Then the element is retrieved - * from the base class (if not null). - * - * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. - */ -@SuppressWarnings("unused") -@Deprecated -public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidthVector, FieldVector { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); - -protected final static byte[] emptyByteArray = new byte[]{}; - - private final String bitsField = "$bits$"; - private final String valuesField = "$values$"; - - final BitVector bits = new BitVector(bitsField, allocator); - final ${valuesName} values; - - private final Mutator mutator; - private final Accessor accessor; - - <#if minor.typeParams??> - <#assign typeParams = minor.typeParams?reverse> - <#list typeParams as typeParam> - private final ${typeParam.type} ${typeParam.name}; - - - /** - * Assumes the type is nullable and not dictionary encoded - * @param name name of the field - * @param allocator allocator to use to resize the vector<#list typeParams as typeParam> - * @param ${typeParam.name} type parameter ${typeParam.name} - */ - public ${className}(String name, BufferAllocator allocator<#list typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { - <#if minor.arrowTypeConstructorParams??> - <#assign constructorParams = minor.arrowTypeConstructorParams /> - <#else> - <#assign constructorParams = [] /> - <#list typeParams as typeParam> - <#assign constructorParams = constructorParams + [ typeParam.name ] /> - - - this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator); - } - <#else> - public ${className}(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator); - } - - - public ${className}(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator); - <#if minor.typeParams??> - <#assign typeParams = minor.typeParams?reverse> - ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType(); - <#list typeParams as typeParam> - this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}(); - - this.values = new ${valuesName}(valuesField, allocator<#list typeParams as typeParam>, ${typeParam.name}); - <#else> - this.values = new ${valuesName}(valuesField, allocator); - - this.mutator = new Mutator(); - this.accessor = new Accessor(); - } - - @Override - public List getFieldInnerVectors() { - /* DELEGATE TO NEW VECTOR */ - return values.getFieldInnerVectors(); - } - - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector ${className} can not have children: " + children); - } - } - - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - /* DELEGATE TO NEW VECTOR */ - values.loadFieldBuffers(fieldNode, ownBuffers); - } - - public List getFieldBuffers() { - /* DELEGATE TO NEW VECTOR */ - return values.getFieldBuffers(); - } - - @Override - public Field getField() { - /* DELEGATE TO NEW VECTOR */ - return values.getField(); - } - - @Override - public MinorType getMinorType() { - /* DELEGATE TO NEW VECTOR */ - return values.getMinorType(); - } - - @Override - public FieldReader getReader(){ - /* DELEGATE TO NEW VECTOR */ - return values.getReader(); - } - - @Override - public int getValueCapacity(){ - /* DELEGATE TO NEW VECTOR */ - return values.getValueCapacity(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - /* DELEGATE TO NEW VECTOR */ - return values.getBuffers(clear); - } - - @Override - public void close() { - /* DELEGATE TO NEW VECTOR */ - values.close(); - } - - @Override - public void clear() { - /* DELEGATE TO NEW VECTOR */ - values.clear(); - } - - @Override - public int getBufferSize(){ - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - /* DELEGATE TO NEW VECTOR */ - return values.getBufferSizeFor(valueCount); - } - - public ArrowBuf getBuffer() { - return values.getDataBuffer(); - } - - public ${valuesName} getValuesVector() { - return values; - } - - @Override - public void setInitialCapacity(int numRecords) { - /* DELEGATE TO NEW VECTOR */ - values.setInitialCapacity(numRecords); - } - - @Override - public void allocateNew() { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - /* DELEGATE TO NEW VECTOR */ - return values.allocateNewSafe(); - } - - @Override - public void reAlloc() { - /* DELEGATE TO NEW VECTOR */ - values.reAlloc(); - } - - public void reset() { - /* DELEGATE TO NEW VECTOR */ - values.reset(); - } - - <#if type.major == "VarLen"> - @Override - public void allocateNew(int totalBytes, int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(totalBytes, valueCount); - } - - @Override - public int getByteCapacity(){ - return values.getByteCapacity(); - } - - @Override - public int getCurrentSizeInBytes(){ - return values.getCurrentSizeInBytes(); - } - - <#else> - @Override - public void allocateNew(int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.allocateNew(valueCount); - } - - /** - * {@inheritDoc} - */ - @Override - public void zeroVector() { - /* DELEGATE TO NEW VECTOR */ - values.zeroVector(); - } - - - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator, callBack); - } - - - - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(allocator); - } - - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - /* DELEGATE TO NEW VECTOR */ - return values.getTransferPair(ref, allocator); - } - - - - @Override - public TransferPair makeTransferPair(ValueVector to) { - /* DELEGATE TO NEW VECTOR */ - return values.makeTransferPair(to); - } - - - - public void transferTo(${valuesName} target) { - /* DELEGATE TO NEW VECTOR */ - <#if type.major == "VarLen"> - values.transferTo((BaseNullableVariableWidthVector) target); - <#else> - values.transferTo((BaseNullableFixedWidthVector) target); - - } - - public void splitAndTransferTo(int startIndex, int length, ${valuesName} target) { - /* DELEGATE TO NEW VECTOR */ - <#if type.major == "VarLen"> - values.splitAndTransferTo(startIndex, length, (BaseNullableVariableWidthVector) target); - <#else> - values.splitAndTransferTo(startIndex, length, (BaseNullableFixedWidthVector) target); - - } - - - - @Override - public Accessor getAccessor(){ - return accessor; - } - - @Override - public Mutator getMutator(){ - return mutator; - } - - - public void copyFrom(int fromIndex, int thisIndex, ${valuesName} from) { - /* DELEGATE TO NEW VECTOR */ - values.copyFrom(fromIndex, thisIndex, from); - } - - public void copyFromSafe(int fromIndex, int thisIndex, ${valuesName} from) { - /* DELEGATE TO NEW VECTOR */ - values.copyFromSafe(fromIndex, thisIndex, from); - } - - @Override - public long getValidityBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getDataBufferAddress(); - } - - @Override - public long getOffsetBufferAddress() { - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBufferAddress(); - } - - @Override - public ArrowBuf getValidityBuffer() { - /* DELEGATE TO NEW VECTOR */ - return values.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - return (values.getDataBuffer()); - } - - @Override - public ArrowBuf getOffsetBuffer() { - /* DELEGATE TO NEW VECTOR */ - return values.getOffsetBuffer(); - } - - public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor { - - /** - * Get the element at the specified position. - * - * @param index position of the value - * @return value of the element, if not null - */ - public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType} get(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.get(index); - } - - @Override - public boolean isNull(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.isNull(index); - } - - public int isSet(int index){ - /* DELEGATE TO NEW VECTOR */ - return values.isSet(index); - } - - <#if type.major == "VarLen"> - public long getStartEnd(int index){ - /* DELEGATE TO NEW VECTOR */ - return values.getStartEnd(index); - } - - @Override - public int getValueLength(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getValueLength(index); - } - - - public void get(int index, Nullable${minor.class}Holder holder){ - /* DELEGATE TO NEW VECTOR */ - values.get(index, holder); - } - - @Override - public ${friendlyType} getObject(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getObject(index); - } - - <#if minor.class == "IntervalYear" || minor.class == "IntervalDay"> - public StringBuilder getAsStringBuilder(int index) { - /* DELEGATE TO NEW VECTOR */ - return values.getAsStringBuilder(index); - } - - - @Override - public int getValueCount(){ - /* DELEGATE TO NEW VECTOR */ - return values.getValueCount(); - } - - public void reset() { } - } - - public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator { - private int setCount; - <#if type.major = "VarLen"> private int lastSet = -1; - - private Mutator() { } - - public ${valuesName} getVectorWithValues() { - return values; - } - - - @Override - public void setIndexDefined(int index) { - /* DELEGATE TO NEW VECTOR */ - values.setIndexDefined(index); - } - - - - /** - * Set the variable length element at the specified index to the supplied byte array. - * - * @param index position of the bit to set - * @param value array of bytes (or int if smaller than 4 bytes) to write - */ - public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType} value) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, value); - } - - - - <#if type.major == "VarLen"> - public void fillEmpties(int index) { - /* DELEGATE TO NEW VECTOR */ - values.fillEmpties(index); - } - - @Override - public void setValueLengthSafe(int index, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setValueLengthSafe(index, length); - } - - - - public void setSafe(int index, byte[] value, int start, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value, start, length); - } - - - public void setSafe(int index, ByteBuffer value, int start, int length) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value, start, length); - } - - - public void setNull(int index) { - /* DELEGATE TO NEW VECTOR */ - values.setNull(index); - } - - - public void set(int index, Nullable${minor.class}Holder holder) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, holder); - } - - - public void set(int index, ${minor.class}Holder holder) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, holder); - } - - - public boolean isSafe(int outIndex) { - /* DELEGATE TO NEW VECTOR */ - return values.isSafe(outIndex); - } - - - <#assign fields = minor.fields!type.fields /> - public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field ){ - values.set(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); - } - - public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field ) { - values.setSafe(index, isSet<#list fields as field><#if field.include!true >, ${field.name}Field); - } - - - public void setSafe(int index, Nullable${minor.class}Holder value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - public void setSafe(int index, ${minor.class}Holder value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - <#if !(type.major == "VarLen" || minor.class == "IntervalDay")> - public void setSafe(int index, ${minor.javaType!type.javaType} value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - - - <#if minor.class == "Decimal"> - public void set(int index, ${friendlyType} value) { - /* DELEGATE TO NEW VECTOR */ - values.set(index, value); - } - - public void setSafe(int index, ${friendlyType} value) { - /* DELEGATE TO NEW VECTOR */ - values.setSafe(index, value); - } - - - - @Override - public void setValueCount(int valueCount) { - /* DELEGATE TO NEW VECTOR */ - values.setValueCount(valueCount); - } - - - /* THIS METHOD IS PROBABLY NOT NEEDED FOR NEW VECTORS */ - @Override - public void generateTestData(int valueCount) { } - - - /* MUTATOR RESET IS NOT NEEDED FOR NEW VECTORS */ - @Override - public void reset() { } - - - <#if type.major == "VarLen"> - public void setLastSet(int value) { - /* DELEGATE TO NEW VECTOR */ - values.setLastSet(value); - } - - - public int getLastSet() { - /* DELEGATE TO NEW VECTOR */ - return values.getLastSet(); - } - - } -} - - diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java index 01340f66c40..38524ff8adf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -85,7 +85,7 @@ public ArrowBuf[] getBuffers(boolean clear) { if (getBufferSize() == 0) { out = new ArrowBuf[0]; } else { - out = new ArrowBuf[] {data}; + out = new ArrowBuf[]{data}; data.readerIndex(0); if (clear) { data.retain(1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index 97dfa247845..a4b48de10a8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,35 +42,35 @@ */ public abstract class BaseNullableFixedWidthVector extends BaseValueVector implements FixedWidthVector, FieldVector, NullableVectorDefinitionSetter { - private final byte typeWidth; - - protected int valueAllocationSizeInBytes; - protected int validityAllocationSizeInBytes; - - protected final Field field; - private int allocationMonitor; - protected ArrowBuf validityBuffer; - protected ArrowBuf valueBuffer; - protected int valueCount; - - public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, - FieldType fieldType, final byte typeWidth) { - super(name, allocator); - this.typeWidth = typeWidth; - field = new Field(name, fieldType, null); - valueCount = 0; - allocationMonitor = 0; - validityBuffer = allocator.getEmpty(); - valueBuffer = allocator.getEmpty(); - if (typeWidth > 0) { - valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - } else { + private final byte typeWidth; + + protected int valueAllocationSizeInBytes; + protected int validityAllocationSizeInBytes; + + protected final Field field; + private int allocationMonitor; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected int valueCount; + + public BaseNullableFixedWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType, final byte typeWidth) { + super(name, allocator); + this.typeWidth = typeWidth; + field = new Field(name, fieldType, null); + valueCount = 0; + allocationMonitor = 0; + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + if (typeWidth > 0) { + valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + } else { /* specialized handling for NullableBitVector */ - valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - validityAllocationSizeInBytes = valueAllocationSizeInBytes; - } - } + valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + validityAllocationSizeInBytes = valueAllocationSizeInBytes; + } + } /* TODO: @@ -93,579 +93,583 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all */ - @Override - @Deprecated - public Mutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); - } - - @Override - @Deprecated - public Accessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - - /** - * Get the memory address of buffer that manages the validity - * (NULL or NON-NULL nature) of elements in the vector. - * @return starting address of the buffer - */ - @Override - public long getValidityBufferAddress() { - return (validityBuffer.memoryAddress()); - } - - /** - * Get the memory address of buffer that stores the data for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getDataBufferAddress() { - return (valueBuffer.memoryAddress()); - } - - /** - * Get the memory address of buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return starting address of the buffer - * @throws UnsupportedOperationException for fixed width vectors - */ - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException("not supported for fixed-width vectors"); - } - - /** - * Get buffer that manages the validity (NULL or NON-NULL nature) of - * elements in the vector. Consider it as a buffer for internal bit vector - * data structure. - * @return buffer - */ - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } - - /** - * Get the buffer that stores the data for elements in the vector. - * @return buffer - */ - @Override - public ArrowBuf getDataBuffer() { - return valueBuffer; - } - - /** - * buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return buffer - * @throws UnsupportedOperationException for fixed width vectors - */ - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException("not supported for fixed-width vectors"); - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final long size = (long)valueCount * typeWidth; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); - } - - private int getValueBufferValueCapacity() { - return (int)((valueBuffer.capacity() * 1.0)/typeWidth); - } - - private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); - } - - /** - * zero out the vector and the data in associated buffers. - */ - @Override - public void zeroVector() { - initValidityBuffer(); - initValueBuffer(); - } - - /* zero out the validity buffer */ - private void initValidityBuffer() { - validityBuffer.setZero(0, validityBuffer.capacity()); - } - - /* zero out the data buffer */ - private void initValueBuffer() { - valueBuffer.setZero(0, valueBuffer.capacity()); - } - - /** - * Reset the vector to initial state. Same as {@link #zeroVector()}. - * Note that this method doesn't release any memory. - */ - public void reset() { - zeroVector(); - } - - /** - * Close the vector and release the associated buffers. - */ - @Override - public void close() { clear(); } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - valueCount = 0; - validityBuffer = releaseBuffer(validityBuffer); - valueBuffer = releaseBuffer(valueBuffer); - } - - /* used to step down the memory allocation */ - protected void incrementAllocationMonitor() { - if (allocationMonitor < 0) { - allocationMonitor = 0; - } - allocationMonitor++; - } - - /* used to step up the memory allocation */ - protected void decrementAllocationMonitor() { - if (allocationMonitor > 0) { - allocationMonitor = 0; - } - allocationMonitor--; - } - - /** - * Same as {@link #allocateNewSafe()}. - */ - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating memory."); - } - } - - /** - * Allocate memory for the vector. We internally use a default value count - * of 4096 to allocate memory for at least these many elements in the - * vector. See {@link #allocateNew(int)} for allocating memory for specific - * number of elements in the vector. - * - * @return false if memory allocation fails, true otherwise. - */ - @Override - public boolean allocateNewSafe() { - long curAllocationSizeValue = valueAllocationSizeInBytes; - long curAllocationSizeValidity = validityAllocationSizeInBytes; + @Override + @Deprecated + public Mutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing to vector"); + } + + @Override + @Deprecated + public Accessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); + } + + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ + @Override + public long getValidityBufferAddress() { + return (validityBuffer.memoryAddress()); + } + + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getDataBufferAddress() { + return (valueBuffer.memoryAddress()); + } + + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return starting address of the buffer + * @throws UnsupportedOperationException for fixed width vectors + */ + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + * @throws UnsupportedOperationException for fixed width vectors + */ + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException("not supported for fixed-width vectors"); + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long) valueCount * typeWidth; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int) size; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return Math.min(getValueBufferValueCapacity(), getValidityBufferValueCapacity()); + } + + private int getValueBufferValueCapacity() { + return (int) ((valueBuffer.capacity() * 1.0) / typeWidth); + } + + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * zero out the vector and the data in associated buffers. + */ + @Override + public void zeroVector() { + initValidityBuffer(); + initValueBuffer(); + } + + /* zero out the validity buffer */ + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + /* zero out the data buffer */ + private void initValueBuffer() { + valueBuffer.setZero(0, valueBuffer.capacity()); + } + + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ + public void reset() { + zeroVector(); + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clear(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + valueCount = 0; + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + } + + /* used to step down the memory allocation */ + protected void incrementAllocationMonitor() { + if (allocationMonitor < 0) { + allocationMonitor = 0; + } + allocationMonitor++; + } - if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } + /* used to step up the memory allocation */ + protected void decrementAllocationMonitor() { + if (allocationMonitor > 0) { + allocationMonitor = 0; + } + allocationMonitor--; + } + + /** + * Same as {@link #allocateNewSafe()}. + */ + @Override + public void allocateNew() { + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } /* we are doing a new allocation -- release the current buffers */ - clear(); + clear(); - try{ - allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); - } catch (Exception e) { - e.printStackTrace(); - clear(); - return false; - } - - return true; - } - - /** - * Allocate memory for the vector to support storing at least the provided number of - * elements in the vector. This method must be called prior to using the ValueVector. - * - * @param valueCount the desired number of elements in the vector - * @throws org.apache.arrow.memory.OutOfMemoryException - */ - public void allocateNew(int valueCount) { - long valueBufferSize = valueCount * typeWidth; - long validityBufferSize = getValidityBufferSizeFromCount(valueCount); - if (typeWidth == 0) { + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity); + } catch (Exception e) { + e.printStackTrace(); + clear(); + return false; + } + + return true; + } + + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ + public void allocateNew(int valueCount) { + long valueBufferSize = valueCount * typeWidth; + long validityBufferSize = getValidityBufferSizeFromCount(valueCount); + if (typeWidth == 0) { /* specialized handling for NullableBitVector */ - valueBufferSize = validityBufferSize; - } + valueBufferSize = validityBufferSize; + } - if (valueBufferSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } + if (valueBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } /* we are doing a new allocation -- release the current buffers */ - clear(); + clear(); - try { - allocateBytes(valueBufferSize, validityBufferSize); - } catch(Exception e) { - e.printStackTrace(); - clear(); - throw e; - } - } - - /** - * Actual memory allocation is done by this function. All the calculations - * and knowledge about what size to allocate is upto the callers of this - * method. - * Callers appropriately handle errors if memory allocation fails here. - * Callers should also take care of determining that desired size is - * within the bounds of max allocation allowed and any other error - * conditions. - */ - private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { + try { + allocateBytes(valueBufferSize, validityBufferSize); + } catch (Exception e) { + e.printStackTrace(); + clear(); + throw e; + } + } + + /** + * Actual memory allocation is done by this function. All the calculations + * and knowledge about what size to allocate is upto the callers of this + * method. + * Callers appropriately handle errors if memory allocation fails here. + * Callers should also take care of determining that desired size is + * within the bounds of max allocation allowed and any other error + * conditions. + */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { /* allocate data buffer */ - int curSize = (int)valueBufferSize; - valueBuffer = allocator.buffer(curSize); - valueBuffer.readerIndex(0); - valueAllocationSizeInBytes = curSize; + int curSize = (int) valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; /* allocate validity buffer */ - allocateValidityBuffer((int)validityBufferSize); - zeroVector(); - } - - /** - * During splitAndTransfer, if we splitting from a random position within a byte, - * we can't just slice the source buffer so we have to explicitly allocate the - * validityBuffer of the target vector. This is unlike the databuffer which we can - * always slice for the target vector. - */ - private void allocateValidityBuffer(final int validityBufferSize) { - validityBuffer = allocator.buffer(validityBufferSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = validityBufferSize; - } - - /** - * Get the potential buffer size for a particular number of records. - * @param count desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int count) { - if (count == 0) { return 0; } - return (count * typeWidth) + getValidityBufferSizeFromCount(count); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - if (valueCount == 0) { return 0; } - return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); - } - - /** - * Get information about how this field is materialized. - * @return the field corresponding to this vector - */ - @Override - public Field getField() { - return field; - } - - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers; - setReaderAndWriterIndex(); - if (getBufferSize() == 0) { - buffers = new ArrowBuf[0]; - } else { - buffers = new ArrowBuf[2]; - buffers[0] = validityBuffer; - buffers[1] = valueBuffer; - } - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - } - - /** - * Resize the vector to increase the capacity. The internal behavior is to - * double the current value capacity. - */ - @Override - public void reAlloc() { - valueBuffer = reallocBufferHelper(valueBuffer, true); - validityBuffer = reallocBufferHelper(validityBuffer, false); - } - - /** - * Helper method for reallocating a particular internal buffer - * Returns the new buffer. - */ - private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { - final int currentBufferCapacity = buffer.capacity(); - long baseSize = (dataBuffer ? valueAllocationSizeInBytes - : validityAllocationSizeInBytes); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); + allocateValidityBuffer((int) validityBufferSize); + zeroVector(); + } + + /** + * During splitAndTransfer, if we splitting from a random position within a byte, + * we can't just slice the source buffer so we have to explicitly allocate the + * validityBuffer of the target vector. This is unlike the databuffer which we can + * always slice for the target vector. + */ + private void allocateValidityBuffer(final int validityBufferSize) { + validityBuffer = allocator.buffer(validityBufferSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = validityBufferSize; + } + + /** + * Get the potential buffer size for a particular number of records. + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { + return 0; + } + return (count * typeWidth) + getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + if (valueCount == 0) { + return 0; + } + return (valueCount * typeWidth) + getValidityBufferSizeFromCount(valueCount); + } + + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ + @Override + public Field getField() { + return field; + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[2]; + buffers[0] = validityBuffer; + buffers[1] = valueBuffer; + } + if (clear) { + for (final ArrowBuf buffer : buffers) { + buffer.retain(1); } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, buffer, 0, currentBufferCapacity); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); - buffer.release(1); - buffer = newBuf; - if (dataBuffer) { - valueAllocationSizeInBytes = (int)newAllocationSize; - } - else { - validityAllocationSizeInBytes = (int)newAllocationSize; - } - - return buffer; - } - - @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); - } - - /** - * Initialize the children in schema for this Field. This operation is a - * NO-OP for scalar types since they don't have any children. - * @param children the schema - * @throws IllegalArgumentException if children is a non-empty list for scalar types. - */ - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector can not have children"); - } - } - - /** - * Get the inner child vectors. - * @return list of child vectors for complex types, empty list for scalar vector - * types - */ - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - /** - * Load the buffers of this vector with provided source buffers. - * The caller manages the source buffers and populates them before invoking - * this method. - * @param fieldNode the fieldNode indicating the value count - * @param ownBuffers the buffers for this Field (own buffers only, children not included) - */ - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - if (ownBuffers.size() != 2) { - throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); - } - - ArrowBuf bitBuffer = ownBuffers.get(0); - ArrowBuf dataBuffer = ownBuffers.get(1); - - validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); - valueBuffer.release(); - valueBuffer = dataBuffer.retain(allocator); - - valueCount = fieldNode.getLength(); - - valueAllocationSizeInBytes = valueBuffer.capacity(); - validityAllocationSizeInBytes = validityBuffer.capacity(); - } - - /** - * Get the buffers belonging to this vector - * @return the inner buffers. - */ - public List getFieldBuffers() { - List result = new ArrayList<>(2); - setReaderAndWriterIndex(); - result.add(validityBuffer); - result.add(valueBuffer); - - return result; - } - - private void setReaderAndWriterIndex() { - validityBuffer.readerIndex(0); - valueBuffer.readerIndex(0); - if (valueCount == 0) { - validityBuffer.writerIndex(0); - valueBuffer.writerIndex(0); - } else { - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - if (typeWidth == 0) { + clear(); + } + return buffers; + } + + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ + @Override + public void reAlloc() { + valueBuffer = reallocBufferHelper(valueBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /** + * Helper method for reallocating a particular internal buffer + * Returns the new buffer. + */ + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean dataBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (dataBuffer ? valueAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (dataBuffer) { + valueAllocationSizeInBytes = (int) newAllocationSize; + } else { + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + return buffer; + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 2) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 2 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf dataBuffer = ownBuffers.get(1); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + valueCount = fieldNode.getLength(); + + valueAllocationSizeInBytes = valueBuffer.capacity(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ + public List getFieldBuffers() { + List result = new ArrayList<>(2); + setReaderAndWriterIndex(); + result.add(validityBuffer); + result.add(valueBuffer); + + return result; + } + + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + valueBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + valueBuffer.writerIndex(0); + } else { + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + if (typeWidth == 0) { /* specialized handling for NullableBitVector */ - valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - } else { - valueBuffer.writerIndex(valueCount * typeWidth); - } + valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + } else { + valueBuffer.writerIndex(valueCount * typeWidth); } - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @param callBack - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return getTransferPair(ref, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param allocator allocator for the target vector - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - return getTransferPair(name, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return TransferPair - */ - public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); - - /** - * Transfer this vector'data to another vector. The memory associated - * with this vector is transferred to the allocator of target vector - * for accounting and management purposes. - * @param target destination vector for transfer - */ - public void transferTo(BaseNullableFixedWidthVector target){ - compareTypes(target, "transferTo"); - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; - target.valueCount = valueCount; - clear(); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - splitAndTransferValueBuffer(startIndex, length, target); - target.setValueCount(length); - } - - /** - * Data buffer can always be split and transferred using slicing. - */ - private void splitAndTransferValueBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { - final int startPoint = startIndex * typeWidth; - final int sliceLength = length * typeWidth; - target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; - } - - /** - * Validity buffer has multiple cases of split and transfer depending on - * the starting position of the source index. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { + } + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ + public void transferTo(BaseNullableFixedWidthVector target) { + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.valueCount = valueCount; + clear(); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferValueBuffer(startIndex, length, target); + target.setValueCount(length); + } + + /** + * Data buffer can always be split and transferred using slicing. + */ + private void splitAndTransferValueBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + final int startPoint = startIndex * typeWidth; + final int sliceLength = length * typeWidth; + target.valueBuffer = valueBuffer.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer; + } + + /** + * Validity buffer has multiple cases of split and transfer depending on + * the starting position of the source index. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { /* slice */ - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } - else { + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. * Each byte in the target is composed by a part in i-th byte, * another part in (i+1)-th byte. */ - target.allocateValidityBuffer(byteSizeTarget); + target.allocateValidityBuffer(byteSizeTarget); - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + i + 1, offset); + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + i + 1, offset); - target.validityBuffer.setByte(i, (b1 + b2)); - } + target.validityBuffer.setByte(i, (b1 + b2)); + } /* Copying the last piece is done in the following manner: * if the source vector has 1 or more bytes remaining, we copy @@ -676,61 +680,60 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - - - /****************************************************************** - * * - * common getters and setters * - * * - ******************************************************************/ - - - /** - * Get the number of elements that are null in the vector - * - * @return the number of null elements. - */ - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - /** - * Get the value count of vector. This will always be zero unless - * {@link #setValueCount(int)} has been called prior to calling this. - * - * @return valueCount for the vector - */ - public int getValueCount(){ - return valueCount; - } - - /** - * Set value count for the vector. - * - * @param valueCount value count to set - */ - public void setValueCount(int valueCount) { - this.valueCount = valueCount; - final int currentValueCapacity = getValueCapacity(); - while (valueCount > getValueCapacity()) { - reAlloc(); + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } } + } + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + /** + * Get the value count of vector. This will always be zero unless + * {@link #setValueCount(int)} has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount() { + return valueCount; + } + + /** + * Set value count for the vector. + * + * @param valueCount value count to set + */ + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + final int currentValueCapacity = getValueCapacity(); + while (valueCount > getValueCapacity()) { + reAlloc(); + } /* * We are trying to understand the pattern of memory allocation. * If initially, the user did vector.allocateNew(), we would have @@ -757,89 +760,89 @@ public void setValueCount(int valueCount) { * This case is also applicable to setSafe() methods which can trigger * a realloc() and thus we record the state there as well. */ - if (valueCount > 0) { - if (currentValueCapacity >= (valueCount * 2)) { - incrementAllocationMonitor(); - } else if (currentValueCapacity <= (valueCount/2)) { - decrementAllocationMonitor(); - } - } - setReaderAndWriterIndex(); - } - - /** - * Check if the given index is within the current value capacity - * of the vector - * - * @param index position to check - * @return true if index is within the current value capacity - */ - public boolean isSafe(int index) { - return index < getValueCapacity(); - } - - /** - * Check if element at given index is null. - * - * @param index position of element - * @return true if element at given index is null, false otherwise - */ - public boolean isNull(int index) { - return (isSet(index) == 0); - } - - /** - * Same as {@link #isNull(int)}. - * - * @param index position of element - * @return 1 if element at given index is not null, 0 otherwise - */ - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Mark the particular position in the vector as non-null. - * - * @param index position of the element. - */ - @Override - public void setIndexDefined(int index) { - handleSafe(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } - - public void set(int index, byte[] value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void setSafe(int index, byte[] value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void set(int index, ByteBuffer value, int start, int length) { - throw new UnsupportedOperationException(); - } - - public void setSafe(int index, ByteBuffer value, int start, int length) { - throw new UnsupportedOperationException(); - } - - - /****************************************************************** - * * - * helper methods for setters * - * * - ******************************************************************/ - - - protected void handleSafe(int index) { - while (index >= getValueCapacity()) { - decrementAllocationMonitor(); - reAlloc(); + if (valueCount > 0) { + if (currentValueCapacity >= (valueCount * 2)) { + incrementAllocationMonitor(); + } else if (currentValueCapacity <= (valueCount / 2)) { + decrementAllocationMonitor(); } - } + } + setReaderAndWriterIndex(); + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ + @Override + public void setIndexDefined(int index) { + handleSafe(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void set(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, byte[] value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void set(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + public void setSafe(int index, ByteBuffer value, int start, int length) { + throw new UnsupportedOperationException(); + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected void handleSafe(int index) { + while (index >= getValueCapacity()) { + decrementAllocationMonitor(); + reAlloc(); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index e08f6e5961e..f819ab1c4e7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,38 +39,38 @@ public abstract class BaseNullableVariableWidthVector extends BaseValueVector implements VariableWidthVector, FieldVector, NullableVectorDefinitionSetter { - private static final int DEFAULT_RECORD_BYTE_COUNT = 8; - private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; - - private int valueAllocationSizeInBytes; - private int validityAllocationSizeInBytes; - private int offsetAllocationSizeInBytes; - - /* protected members */ - public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ - protected static final byte[] emptyByteArray = new byte[]{}; - protected ArrowBuf validityBuffer; - protected ArrowBuf valueBuffer; - protected ArrowBuf offsetBuffer; - protected int valueCount; - protected int lastSet; - protected final Field field; - private boolean cleared; - - public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, - FieldType fieldType) { - super(name, allocator); - valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); - offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; - field = new Field(name, fieldType, null); - valueCount = 0; - lastSet = -1; - offsetBuffer = allocator.getEmpty(); - validityBuffer = allocator.getEmpty(); - valueBuffer = allocator.getEmpty(); - cleared = false; - } + private static final int DEFAULT_RECORD_BYTE_COUNT = 8; + private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT; + + private int valueAllocationSizeInBytes; + private int validityAllocationSizeInBytes; + private int offsetAllocationSizeInBytes; + + /* protected members */ + public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */ + protected static final byte[] emptyByteArray = new byte[]{}; + protected ArrowBuf validityBuffer; + protected ArrowBuf valueBuffer; + protected ArrowBuf offsetBuffer; + protected int valueCount; + protected int lastSet; + protected final Field field; + private boolean cleared; + + public BaseNullableVariableWidthVector(final String name, final BufferAllocator allocator, + FieldType fieldType) { + super(name, allocator); + valueAllocationSizeInBytes = INITIAL_BYTE_COUNT; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); + offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH; + field = new Field(name, fieldType, null); + valueCount = 0; + lastSet = -1; + offsetBuffer = allocator.getEmpty(); + validityBuffer = allocator.getEmpty(); + valueBuffer = allocator.getEmpty(); + cleared = false; + } /* TODO: * @@ -92,658 +92,656 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator */ - @Override - @Deprecated - public VariableWidthMutator getMutator() { - throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); - } - - @Override - @Deprecated - public VariableWidthAccessor getAccessor() { - throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); - } - - /** - * Get buffer that manages the validity (NULL or NON-NULL nature) of - * elements in the vector. Consider it as a buffer for internal bit vector - * data structure. - * @return buffer - */ - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } - - /** - * Get the buffer that stores the data for elements in the vector. - * @return buffer - */ - @Override - public ArrowBuf getDataBuffer() { - return valueBuffer; - } - - /** - * buffer that stores the offsets for elements - * in the vector. This operation is not supported for fixed-width vectors. - * @return buffer - */ - @Override - public ArrowBuf getOffsetBuffer() { - return offsetBuffer; - } - - /** - * Get the memory address of buffer that stores the offsets for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getOffsetBufferAddress() { - return offsetBuffer.memoryAddress(); - } - - /** - * Get the memory address of buffer that manages the validity - * (NULL or NON-NULL nature) of elements in the vector. - * @return starting address of the buffer - */ - @Override - public long getValidityBufferAddress() { - return validityBuffer.memoryAddress(); - } - - /** - * Get the memory address of buffer that stores the data for elements - * in the vector. - * @return starting address of the buffer - */ - @Override - public long getDataBufferAddress() { - return valueBuffer.memoryAddress(); - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final long size = (long)valueCount * DEFAULT_RECORD_BYTE_COUNT; - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = (int)size; - validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); + @Override + @Deprecated + public VariableWidthMutator getMutator() { + throw new UnsupportedOperationException("Mutator is not supported for writing into vector"); + } + + @Override + @Deprecated + public VariableWidthAccessor getAccessor() { + throw new UnsupportedOperationException("Accessor is not supported for reading from vector"); + } + + /** + * Get buffer that manages the validity (NULL or NON-NULL nature) of + * elements in the vector. Consider it as a buffer for internal bit vector + * data structure. + * @return buffer + */ + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + /** + * Get the buffer that stores the data for elements in the vector. + * @return buffer + */ + @Override + public ArrowBuf getDataBuffer() { + return valueBuffer; + } + + /** + * buffer that stores the offsets for elements + * in the vector. This operation is not supported for fixed-width vectors. + * @return buffer + */ + @Override + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } + + /** + * Get the memory address of buffer that stores the offsets for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getOffsetBufferAddress() { + return offsetBuffer.memoryAddress(); + } + + /** + * Get the memory address of buffer that manages the validity + * (NULL or NON-NULL nature) of elements in the vector. + * @return starting address of the buffer + */ + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + /** + * Get the memory address of buffer that stores the data for elements + * in the vector. + * @return starting address of the buffer + */ + @Override + public long getDataBufferAddress() { + return valueBuffer.memoryAddress(); + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final long size = (long) valueCount * DEFAULT_RECORD_BYTE_COUNT; + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = (int) size; + validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); /* to track the end offset of last data element in vector, we need * an additional slot in offset buffer. */ - offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; - } + offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); + return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); + } + + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + private int getOffsetBufferValueCapacity() { + return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH); + } + + /** + * zero out the vector and the data in associated buffers. + */ + public void zeroVector() { + initValidityBuffer(); + initOffsetBuffer(); + } + + /* zero out the validity buffer */ + private void initValidityBuffer() { + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + /* zero out the offset buffer */ + private void initOffsetBuffer() { + offsetBuffer.setZero(0, offsetBuffer.capacity()); + } + + /** + * Reset the vector to initial state. Same as {@link #zeroVector()}. + * Note that this method doesn't release any memory. + */ + public void reset() { + zeroVector(); + lastSet = -1; + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clear(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + validityBuffer = releaseBuffer(validityBuffer); + valueBuffer = releaseBuffer(valueBuffer); + offsetBuffer = releaseBuffer(offsetBuffer); + cleared = true; + lastSet = -1; + valueCount = 0; + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + /** + * Initialize the children in schema for this Field. This operation is a + * NO-OP for scalar types since they don't have any children. + * @param children the schema + * @throws IllegalArgumentException if children is a non-empty list for scalar types. + */ + @Override + public void initializeChildrenFromFields(List children) { + if (!children.isEmpty()) { + throw new IllegalArgumentException("primitive type vector can not have children"); + } + } + + /** + * Get the inner child vectors. + * @return list of child vectors for complex types, empty list for scalar vector + * types + */ + @Override + public List getChildrenFromFields() { + return Collections.emptyList(); + } + + + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + ArrowBuf bitBuffer = ownBuffers.get(0); + ArrowBuf offBuffer = ownBuffers.get(1); + ArrowBuf dataBuffer = ownBuffers.get(2); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + offsetBuffer.release(); + offsetBuffer = offBuffer.retain(allocator); + valueBuffer.release(); + valueBuffer = dataBuffer.retain(allocator); + + lastSet = fieldNode.getLength() - 1; + valueCount = fieldNode.getLength(); + } + + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ + public List getFieldBuffers() { + List result = new ArrayList<>(3); + setReaderAndWriterIndex(); + result.add(validityBuffer); + result.add(offsetBuffer); + result.add(valueBuffer); + + return result; + } + + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + offsetBuffer.readerIndex(0); + valueBuffer.readerIndex(0); + if (valueCount == 0) { + validityBuffer.writerIndex(0); + offsetBuffer.writerIndex(0); + valueBuffer.writerIndex(0); + } else { + final int lastDataOffset = getstartOffset(valueCount); + validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); + offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); + valueBuffer.writerIndex(lastDataOffset); + } + } + + /** + * Same as {@link #allocateNewSafe()}. + */ + @Override + public void allocateNew() { + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory."); + } + } + + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. See {@link #allocateNew(int, int)} for allocating memory for specific + * number of elements in the vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override + public boolean allocateNewSafe() { + long curAllocationSizeValue = valueAllocationSizeInBytes; + long curAllocationSizeValidity = validityAllocationSizeInBytes; + long curAllocationSizeOffset = offsetAllocationSizeInBytes; + + if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || + curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); - return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); - } - - private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); - } - - private int getOffsetBufferValueCapacity() { - return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); - } - - /** - * zero out the vector and the data in associated buffers. - */ - public void zeroVector() { - initValidityBuffer(); - initOffsetBuffer(); - } - - /* zero out the validity buffer */ - private void initValidityBuffer() { - validityBuffer.setZero(0, validityBuffer.capacity()); - } - - /* zero out the offset buffer */ - private void initOffsetBuffer() { - offsetBuffer.setZero(0, offsetBuffer.capacity()); - } - - /** - * Reset the vector to initial state. Same as {@link #zeroVector()}. - * Note that this method doesn't release any memory. - */ - public void reset() { - zeroVector(); - lastSet = -1; - } + /* we are doing a new allocation -- release the current buffers */ + clear(); - /** - * Close the vector and release the associated buffers. - */ - @Override - public void close() { + try { + allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); + } catch (Exception e) { + e.printStackTrace(); clear(); - } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - validityBuffer = releaseBuffer(validityBuffer); - valueBuffer = releaseBuffer(valueBuffer); - offsetBuffer = releaseBuffer(offsetBuffer); - cleared = true; - lastSet = -1; - valueCount = 0; - } - - @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); - } - - /** - * Initialize the children in schema for this Field. This operation is a - * NO-OP for scalar types since they don't have any children. - * @param children the schema - * @throws IllegalArgumentException if children is a non-empty list for scalar types. - */ - @Override - public void initializeChildrenFromFields(List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("primitive type vector can not have children"); - } - } - - /** - * Get the inner child vectors. - * @return list of child vectors for complex types, empty list for scalar vector - * types - */ - @Override - public List getChildrenFromFields() { - return Collections.emptyList(); - } - - - /** - * Load the buffers of this vector with provided source buffers. - * The caller manages the source buffers and populates them before invoking - * this method. - * @param fieldNode the fieldNode indicating the value count - * @param ownBuffers the buffers for this Field (own buffers only, children not included) - */ - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - ArrowBuf bitBuffer = ownBuffers.get(0); - ArrowBuf offBuffer = ownBuffers.get(1); - ArrowBuf dataBuffer = ownBuffers.get(2); - - validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); - offsetBuffer.release(); - offsetBuffer = offBuffer.retain(allocator); - valueBuffer.release(); - valueBuffer = dataBuffer.retain(allocator); - - lastSet = fieldNode.getLength() - 1; - valueCount = fieldNode.getLength(); - } - - /** - * Get the buffers belonging to this vector - * @return the inner buffers. - */ - public List getFieldBuffers() { - List result = new ArrayList<>(3); - setReaderAndWriterIndex(); - result.add(validityBuffer); - result.add(offsetBuffer); - result.add(valueBuffer); - - return result; - } - - private void setReaderAndWriterIndex() { - validityBuffer.readerIndex(0); - offsetBuffer.readerIndex(0); - valueBuffer.readerIndex(0); - if (valueCount == 0) { - validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); - valueBuffer.writerIndex(0); - } else { - final int lastDataOffset = getstartOffset(valueCount); - validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); - valueBuffer.writerIndex(lastDataOffset); - } - } - - /** - * Same as {@link #allocateNewSafe()}. - */ - @Override - public void allocateNew() { - if(!allocateNewSafe()){ - throw new OutOfMemoryException("Failure while allocating memory."); - } - } - - /** - * Allocate memory for the vector. We internally use a default value count - * of 4096 to allocate memory for at least these many elements in the - * vector. See {@link #allocateNew(int, int)} for allocating memory for specific - * number of elements in the vector. - * - * @return false if memory allocation fails, true otherwise. - */ - @Override - public boolean allocateNewSafe() { - long curAllocationSizeValue = valueAllocationSizeInBytes; - long curAllocationSizeValidity = validityAllocationSizeInBytes; - long curAllocationSizeOffset = offsetAllocationSizeInBytes; - - if (curAllocationSizeValue > MAX_ALLOCATION_SIZE || - curAllocationSizeOffset > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } + return false; + } + + return true; + } + + /** + * Allocate memory for the vector to support storing at least the provided number of + * elements in the vector. This method must be called prior to using the ValueVector. + * + * @param totalBytes desired total memory capacity + * @param valueCount the desired number of elements in the vector + * @throws org.apache.arrow.memory.OutOfMemoryException + */ + @Override + public void allocateNew(int totalBytes, int valueCount) { + assert totalBytes >= 0; + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + + if (totalBytes > MAX_ALLOCATION_SIZE || + offsetBufferSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory exceeds limit"); + } /* we are doing a new allocation -- release the current buffers */ - clear(); + clear(); - try { - allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset); - } catch (Exception e) { - e.printStackTrace(); - clear(); - return false; - } - - return true; - } - - /** - * Allocate memory for the vector to support storing at least the provided number of - * elements in the vector. This method must be called prior to using the ValueVector. - * - * @param totalBytes desired total memory capacity - * @param valueCount the desired number of elements in the vector - * @throws org.apache.arrow.memory.OutOfMemoryException - */ - @Override - public void allocateNew(int totalBytes, int valueCount) { - assert totalBytes >= 0; - final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); - - if (totalBytes > MAX_ALLOCATION_SIZE || - offsetBufferSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory exceeds limit"); - } - - /* we are doing a new allocation -- release the current buffers */ + try { + allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); + } catch (Exception e) { + e.printStackTrace(); clear(); + } + } - try { - allocateBytes(totalBytes, validityBufferSize, offsetBufferSize); - } catch (Exception e) { - e.printStackTrace(); - clear(); - } - } - - /* allocate the inner buffers */ - private void allocateBytes(final long valueBufferSize, final long validityBufferSize, - final long offsetBufferSize) { + /* allocate the inner buffers */ + private void allocateBytes(final long valueBufferSize, final long validityBufferSize, + final long offsetBufferSize) { /* allocate data buffer */ - int curSize = (int)valueBufferSize; - valueBuffer = allocator.buffer(curSize); - valueBuffer.readerIndex(0); - valueAllocationSizeInBytes = curSize; - allocateValidityBuffer(validityBufferSize); - allocateOffsetBuffer(offsetBufferSize); - } - - /* allocate offset buffer */ - private void allocateOffsetBuffer(final long size) { - final int curSize = (int)size; - offsetBuffer = allocator.buffer(curSize); - offsetBuffer.readerIndex(0); - offsetAllocationSizeInBytes = curSize; - initOffsetBuffer(); - } - - /* allocate validity buffer */ - private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - initValidityBuffer(); - } - - /** - * Resize the vector to increase the capacity. The internal behavior is to - * double the current value capacity. - */ - public void reAlloc() { - reallocDataBuffer(); - reallocValidityAndOffsetBuffers(); - } - - /** - * Reallocate the data buffer. Data Buffer stores the actual data for - * VARCHAR or VARBINARY elements in the vector. The behavior is to double - * the size of buffer. - * @throws OversizedAllocationException if the desired new size is more than - * max allowed - * @throws OutOfMemoryException if the internal memory allocation fails - */ - public void reallocDataBuffer() { - long baseSize = valueAllocationSizeInBytes; - final int currentBufferCapacity = valueBuffer.capacity(); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); - valueBuffer.release(); - valueBuffer = newBuf; - valueAllocationSizeInBytes = (int)newAllocationSize; - } - - /** - * Reallocate the validity and offset buffers for this vector. Validity - * buffer is used to track the NULL or NON-NULL nature of elements in - * the vector and offset buffer is used to store the lengths of variable - * width elements in the vector. - * - * Note that data buffer for variable length vectors moves independent - * of the companion validity and offset buffers. This is in - * contrast to what we have for fixed width vectors. - * - * So even though we may have setup an initial capacity of 1024 - * elements in the vector, it is quite possible - * that we need to reAlloc() the data buffer when we are setting - * the 5th element in the vector simply because previous - * variable length elements have exhausted the buffer capacity. - * However, we really don't need to reAlloc() validity and - * offset buffers until we try to set the 1025th element - * This is why we do a separate check for safe methods to - * determine which buffer needs reallocation. - * @throws OversizedAllocationException if the desired new size is more than - * max allowed - * @throws OutOfMemoryException if the internal memory allocation fails - */ - public void reallocValidityAndOffsetBuffers() { - offsetBuffer = reallocBufferHelper(offsetBuffer, true); - validityBuffer = reallocBufferHelper(validityBuffer, false); - } - - /* helper method to realloc a particular buffer. returns the allocated buffer */ - private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { - final int currentBufferCapacity = buffer.capacity(); - long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes - : validityAllocationSizeInBytes); - - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); - newBuf.setBytes(0, buffer, 0, currentBufferCapacity); - final int halfNewCapacity = newBuf.capacity() / 2; - newBuf.setZero(halfNewCapacity, halfNewCapacity); - buffer.release(1); - buffer = newBuf; - if (offsetBuffer) { - offsetAllocationSizeInBytes = (int)newAllocationSize; - } - else { - validityAllocationSizeInBytes = (int)newAllocationSize; - } - - return buffer; - } - - /** - * Get the size (number of bytes) of underlying data buffer. - * @return - */ - @Override - public int getByteCapacity(){ - return valueBuffer.capacity(); - } - - @Override - public int getCurrentSizeInBytes(){ + int curSize = (int) valueBufferSize; + valueBuffer = allocator.buffer(curSize); + valueBuffer.readerIndex(0); + valueAllocationSizeInBytes = curSize; + allocateValidityBuffer(validityBufferSize); + allocateOffsetBuffer(offsetBufferSize); + } + + /* allocate offset buffer */ + private void allocateOffsetBuffer(final long size) { + final int curSize = (int) size; + offsetBuffer = allocator.buffer(curSize); + offsetBuffer.readerIndex(0); + offsetAllocationSizeInBytes = curSize; + initOffsetBuffer(); + } + + /* allocate validity buffer */ + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + initValidityBuffer(); + } + + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ + public void reAlloc() { + reallocDataBuffer(); + reallocValidityAndOffsetBuffers(); + } + + /** + * Reallocate the data buffer. Data Buffer stores the actual data for + * VARCHAR or VARBINARY elements in the vector. The behavior is to double + * the size of buffer. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ + public void reallocDataBuffer() { + long baseSize = valueAllocationSizeInBytes; + final int currentBufferCapacity = valueBuffer.capacity(); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity); + valueBuffer.release(); + valueBuffer = newBuf; + valueAllocationSizeInBytes = (int) newAllocationSize; + } + + /** + * Reallocate the validity and offset buffers for this vector. Validity + * buffer is used to track the NULL or NON-NULL nature of elements in + * the vector and offset buffer is used to store the lengths of variable + * width elements in the vector. + * + * Note that data buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + * @throws OversizedAllocationException if the desired new size is more than + * max allowed + * @throws OutOfMemoryException if the internal memory allocation fails + */ + public void reallocValidityAndOffsetBuffers() { + offsetBuffer = reallocBufferHelper(offsetBuffer, true); + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + + /* helper method to realloc a particular buffer. returns the allocated buffer */ + private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) { + final int currentBufferCapacity = buffer.capacity(); + long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes + : validityAllocationSizeInBytes); + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setBytes(0, buffer, 0, currentBufferCapacity); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); + buffer.release(1); + buffer = newBuf; + if (offsetBuffer) { + offsetAllocationSizeInBytes = (int) newAllocationSize; + } else { + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + return buffer; + } + + /** + * Get the size (number of bytes) of underlying data buffer. + * @return + */ + @Override + public int getByteCapacity() { + return valueBuffer.capacity(); + } + + @Override + public int getCurrentSizeInBytes() { /* TODO */ + return 0; + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(this.valueCount); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { return 0; - } + } - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - return getBufferSizeFor(this.valueCount); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param valueCount desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - - final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); - final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; + final int validityBufferSize = getValidityBufferSizeFromCount(valueCount); + final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH; /* get the end offset for this valueCount */ - final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); - return validityBufferSize + offsetBufferSize + dataBufferSize; - } - - /** - * Get information about how this field is materialized. - * @return the field corresponding to this vector - */ - @Override - public Field getField() { - return field; - } - - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers; - setReaderAndWriterIndex(); - if (getBufferSize() == 0) { - buffers = new ArrowBuf[0]; - } else { - buffers = new ArrowBuf[3]; - buffers[0] = validityBuffer; - buffers[1] = offsetBuffer; - buffers[2] = valueBuffer; - } - if (clear) { - for (final ArrowBuf buffer:buffers) { - buffer.retain(1); - } - clear(); - } - return buffers; - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @param callBack - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return getTransferPair(ref, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param allocator allocator for the target vector - * @return TransferPair - */ - @Override - public TransferPair getTransferPair(BufferAllocator allocator){ - return getTransferPair(name, allocator); - } - - /** - * Construct a transfer pair of this vector and another vector of same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return TransferPair - */ - public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); - - /** - * Transfer this vector'data to another vector. The memory associated - * with this vector is transferred to the allocator of target vector - * for accounting and management purposes. - * @param target destination vector for transfer - */ - public void transferTo(BaseNullableVariableWidthVector target){ - compareTypes(target, "transferTo"); - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; - target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; - target.setLastSet(this.lastSet); - if (this.valueCount > 0) { - target.setValueCount(this.valueCount); + final int dataBufferSize = offsetBuffer.getInt(valueCount * OFFSET_WIDTH); + return validityBufferSize + offsetBufferSize + dataBufferSize; + } + + /** + * Get information about how this field is materialized. + * @return the field corresponding to this vector + */ + @Override + public Field getField() { + return field; + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + final ArrowBuf[] buffers; + setReaderAndWriterIndex(); + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = new ArrowBuf[3]; + buffers[0] = validityBuffer; + buffers[1] = offsetBuffer; + buffers[2] = valueBuffer; + } + if (clear) { + for (final ArrowBuf buffer : buffers) { + buffer.retain(1); } clear(); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableVariableWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - splitAndTransferOffsetBuffer(startIndex, length, target); - target.setLastSet(length - 1); - if (this.valueCount > 0) { - target.setValueCount(this.valueCount); - } - } - - /* - * Transfer the offsets along with data. Unlike the data buffer, we cannot simply - * slice the offset buffer for split and transfer. The reason is that offsets - * in the target vector have to be adjusted and made relative to the staring - * offset in source vector from the start index of split. This is why, we - * need to explicitly allocate the offset buffer and set the adjusted offsets - * in the target vector. - */ - private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { - final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); - final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); - final int dataLength = end - start; - target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); - for (int i = 0; i < length + 1; i++) { - final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; - target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); - } - target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; - } - - /* - * Transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, - BaseNullableVariableWidthVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } - else { + } + return buffers; + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @param callBack + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return getTransferPair(ref, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param allocator allocator for the target vector + * @return TransferPair + */ + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return getTransferPair(name, allocator); + } + + /** + * Construct a transfer pair of this vector and another vector of same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return TransferPair + */ + public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); + + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + * @param target destination vector for transfer + */ + public void transferTo(BaseNullableVariableWidthVector target) { + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + target.valueBuffer = valueBuffer.transferOwnership(target.allocator).buffer; + target.offsetBuffer = offsetBuffer.transferOwnership(target.allocator).buffer; + target.setLastSet(this.lastSet); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } + clear(); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableVariableWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferOffsetBuffer(startIndex, length, target); + target.setLastSet(length - 1); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } + } + + /* + * Transfer the offsets along with data. Unlike the data buffer, we cannot simply + * slice the offset buffer for split and transfer. The reason is that offsets + * in the target vector have to be adjusted and made relative to the staring + * offset in source vector from the start index of split. This is why, we + * need to explicitly allocate the offset buffer and set the adjusted offsets + * in the target vector. + */ + private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseNullableVariableWidthVector target) { + final int start = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); + final int end = offsetBuffer.getInt((startIndex + length) * OFFSET_WIDTH); + final int dataLength = end - start; + target.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + for (int i = 0; i < length + 1; i++) { + final int relativeSourceOffset = offsetBuffer.getInt((startIndex + i) * OFFSET_WIDTH) - start; + target.offsetBuffer.setInt(i * OFFSET_WIDTH, relativeSourceOffset); + } + target.valueBuffer = valueBuffer.slice(start, dataLength).transferOwnership(target.allocator).buffer; + } + + /* + * Transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseNullableVariableWidthVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. * Each byte in the target is composed by a part in i-th byte, * another part in (i+1)-th byte. */ - target.allocateValidityBuffer(byteSizeTarget); + target.allocateValidityBuffer(byteSizeTarget); - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); - target.validityBuffer.setByte(i, (b1 + b2)); - } + target.validityBuffer.setByte(i, (b1 + b2)); + } /* Copying the last piece is done in the following manner: * if the source vector has 1 or more bytes remaining, we copy @@ -754,427 +752,426 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - } - - - /****************************************************************** - * * - * common getters and setters * - * * - ******************************************************************/ - - - /** - * Get the number of elements that are null in the vector - * - * @return the number of null elements. - */ - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - /** - * Check if the given index is within the current value capacity - * of the vector - * - * @param index position to check - * @return true if index is within the current value capacity - */ - public boolean isSafe(int index) { - return index < getValueCapacity(); - } - - /** - * Check if element at given index is null. - * - * @param index position of element - * @return true if element at given index is null - */ - public boolean isNull(int index) { - return (isSet(index) == 0); - } - - /** - * Same as {@link #isNull(int)}. - * - * @param index position of element - * @return 1 if element at given index is not null, 0 otherwise - */ - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Get the value count of vector. This will always be zero unless - * setValueCount(int) has been called prior to calling this. - * - * @return valueCount for the vector - */ - public int getValueCount(){ - return valueCount; - } - - /** - * Sets the value count for the vector - * - * @param valueCount value count - */ - public void setValueCount(int valueCount) { - assert valueCount >= 0; - this.valueCount = valueCount; - while (valueCount > getValueCapacity()) { - reallocValidityAndOffsetBuffers(); - } - fillHoles(valueCount); - lastSet = valueCount - 1; - setReaderAndWriterIndex(); - } - - /** - * Create holes in the vector upto the given index (exclusive). - * Holes will be created from the current last set position in - * the vector. - * - * @param index target index - */ - public void fillEmpties(int index) { - handleSafe(index, emptyByteArray.length); - fillHoles(index); - lastSet = index - 1; - } - - /** - * Set the index of last non-null element in the vector. - * It is important to call this method with appropriate value - * before calling {@link #setValueCount(int)}. - * - * @param value desired index of last non-null element. - */ - public void setLastSet(int value) { - lastSet = value; - } - - /** - * Get the index of last non-null element in the vector. - * - * @return index of the last non-null element - */ - public int getLastSet() { - return lastSet; - } - - /** - * Get the starting position (offset) in the data stream for a given - * element in the vector. - * - * @param index position of the element in the vector - * @return starting offset for the element - */ - public long getStartEnd(int index) { - return offsetBuffer.getLong(index * OFFSET_WIDTH); - } - - /** - * Mark the particular position in the vector as non-null. - * - * @param index position of the element. - */ - @Override - public void setIndexDefined(int index) { - while (index >= getValidityBufferValueCapacity()) { - validityBuffer = reallocBufferHelper(validityBuffer, false); - } - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } - - /** - * Sets the value length for an element. - * - * @param index position of the element to set - * @param length length of the element - */ - public void setValueLengthSafe(int index, int length) { - assert index >= 0; - handleSafe(index, length); - fillHoles(index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - lastSet = index; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return greater than 0 length for non-null element, 0 otherwise - */ - public int getValueLength(int index) { - assert index >= 0; - if(isSet(index) == 0) { - return 0; - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - return dataLength; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. This is same as using {@link #set(int, byte[], int, int)} - * with start as 0 and length as value.length - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void set(int index, byte[] value) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[])} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - */ - public void setSafe(int index, byte[] value) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, value.length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, 0, value.length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the supplied - * byte array. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void set(int index, byte[] value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, byte[], int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value array of bytes to write - * @param start start index in array of bytes - * @param length length of data in array of bytes - */ - public void setSafe(int index, byte[] value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setBytes(index, value, start, length); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the - * content in supplied ByteBuffer - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void set(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param value ByteBuffer with data - * @param start start index in ByteBuffer - * @param length length of data in ByteBuffer - */ - public void setSafe(int index, ByteBuffer value, int start, int length) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - valueBuffer.setBytes(startOffset, value, start, length); - lastSet = index; - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - while (index >= getValidityBufferValueCapacity()) { - validityBuffer = reallocBufferHelper(validityBuffer, false); + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } } - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - final int dataLength = end - start; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, buffer, start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of data in buffer - * @param end end position of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { - assert index >= 0; - final int dataLength = end - start; - fillEmpties(index); - handleSafe(index, end); - BitVectorHelper.setValidityBit(validityBuffer, index, isSet); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, buffer, start, dataLength); - lastSet = index; - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param start start position of data in buffer - * @param length length of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void set(int index, int start, int length, ArrowBuf buffer) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - final ArrowBuf bb = buffer.slice(start, length); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - /** - * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param start start position of data in buffer - * @param length length of data in buffer - * @param buffer data buffer containing the variable width element to be stored - * in the vector - */ - public void setSafe(int index, int start, int length, ArrowBuf buffer) { - assert index >= 0; - fillEmpties(index); - handleSafe(index, length); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - final ArrowBuf bb = buffer.slice(start, length); - valueBuffer.setBytes(startOffset, bb); - lastSet = index; - } - - - /****************************************************************** - * * - * helper methods for setters * - * * - ******************************************************************/ - - - protected final void fillHoles(int index) { - for (int i = lastSet + 1; i < index; i++) { - setBytes(i, emptyByteArray, 0, emptyByteArray.length); - } - lastSet = index - 1; - } - - protected final void setBytes(int index, byte[] value, int start, int length) { + } + } + + + /****************************************************************** + * * + * common getters and setters * + * * + ******************************************************************/ + + + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + /** + * Check if the given index is within the current value capacity + * of the vector + * + * @param index position to check + * @return true if index is within the current value capacity + */ + public boolean isSafe(int index) { + return index < getValueCapacity(); + } + + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null + */ + public boolean isNull(int index) { + return (isSet(index) == 0); + } + + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the value count of vector. This will always be zero unless + * setValueCount(int) has been called prior to calling this. + * + * @return valueCount for the vector + */ + public int getValueCount() { + return valueCount; + } + + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ + public void setValueCount(int valueCount) { + assert valueCount >= 0; + this.valueCount = valueCount; + while (valueCount > getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + fillHoles(valueCount); + lastSet = valueCount - 1; + setReaderAndWriterIndex(); + } + + /** + * Create holes in the vector upto the given index (exclusive). + * Holes will be created from the current last set position in + * the vector. + * + * @param index target index + */ + public void fillEmpties(int index) { + handleSafe(index, emptyByteArray.length); + fillHoles(index); + lastSet = index - 1; + } + + /** + * Set the index of last non-null element in the vector. + * It is important to call this method with appropriate value + * before calling {@link #setValueCount(int)}. + * + * @param value desired index of last non-null element. + */ + public void setLastSet(int value) { + lastSet = value; + } + + /** + * Get the index of last non-null element in the vector. + * + * @return index of the last non-null element + */ + public int getLastSet() { + return lastSet; + } + + /** + * Get the starting position (offset) in the data stream for a given + * element in the vector. + * + * @param index position of the element in the vector + * @return starting offset for the element + */ + public long getStartEnd(int index) { + return offsetBuffer.getLong(index * OFFSET_WIDTH); + } + + /** + * Mark the particular position in the vector as non-null. + * + * @param index position of the element. + */ + @Override + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + /** + * Sets the value length for an element. + * + * @param index position of the element to set + * @param length length of the element + */ + public void setValueLengthSafe(int index, int length) { + assert index >= 0; + handleSafe(index, length); + fillHoles(index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + lastSet = index; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return greater than 0 length for non-null element, 0 otherwise + */ + public int getValueLength(int index) { + assert index >= 0; + if (isSet(index) == 0) { + return 0; + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + return dataLength; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. This is same as using {@link #set(int, byte[], int, int)} + * with start as 0 and length as value.length + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void set(int index, byte[] value) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[])} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + */ + public void setSafe(int index, byte[] value) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, value.length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, 0, value.length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the supplied + * byte array. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void set(int index, byte[] value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, byte[], int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value array of bytes to write + * @param start start index in array of bytes + * @param length length of data in array of bytes + */ + public void setSafe(int index, byte[] value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setBytes(index, value, start, length); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the + * content in supplied ByteBuffer + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void set(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Same as {@link #set(int, ByteBuffer, int, int)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param value ByteBuffer with data + * @param start start index in ByteBuffer + * @param length length of data in ByteBuffer + */ + public void setSafe(int index, ByteBuffer value, int start, int length) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + valueBuffer.setBytes(startOffset, value, start, length); + lastSet = index; + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + validityBuffer = reallocBufferHelper(validityBuffer, false); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of data in buffer + * @param end end position of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int isSet, int start, int end, ArrowBuf buffer) { + assert index >= 0; + final int dataLength = end - start; + fillEmpties(index); + handleSafe(index, end); + BitVectorHelper.setValidityBit(validityBuffer, index, isSet); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, buffer, start, dataLength); + lastSet = index; + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void set(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + /** + * Same as {@link #set(int, int, int, int, ArrowBuf)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param start start position of data in buffer + * @param length length of data in buffer + * @param buffer data buffer containing the variable width element to be stored + * in the vector + */ + public void setSafe(int index, int start, int length, ArrowBuf buffer) { + assert index >= 0; + fillEmpties(index); + handleSafe(index, length); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = offsetBuffer.getInt(index * OFFSET_WIDTH); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + final ArrowBuf bb = buffer.slice(start, length); + valueBuffer.setBytes(startOffset, bb); + lastSet = index; + } + + + /****************************************************************** + * * + * helper methods for setters * + * * + ******************************************************************/ + + + protected final void fillHoles(int index) { + for (int i = lastSet + 1; i < index; i++) { + setBytes(i, emptyByteArray, 0, emptyByteArray.length); + } + lastSet = index - 1; + } + + protected final void setBytes(int index, byte[] value, int start, int length) { /* end offset of current last element in the vector. this will * be the start offset of new element we are trying to store. */ - final int startOffset = getstartOffset(index); + final int startOffset = getstartOffset(index); /* set new end offset */ - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); /* store the var length data in value buffer */ - valueBuffer.setBytes(startOffset, value, start, length); - } + valueBuffer.setBytes(startOffset, value, start, length); + } - protected final int getstartOffset(int index) { - return offsetBuffer.getInt(index * OFFSET_WIDTH); - } + protected final int getstartOffset(int index) { + return offsetBuffer.getInt(index * OFFSET_WIDTH); + } - protected final void handleSafe(int index, int dataLength) { + protected final void handleSafe(int index, int dataLength) { /* * IMPORTANT: * value buffer for variable length vectors moves independent @@ -1195,59 +1192,59 @@ protected final void handleSafe(int index, int dataLength) { * This is why we do a separate check for safe methods to * determine which buffer needs reallocation. */ - while (index >= getValueCapacity()) { - reallocValidityAndOffsetBuffers(); - } - final int startOffset = getstartOffset(index); - while (valueBuffer.capacity() < (startOffset + dataLength)) { - reallocDataBuffer(); - } - } - - /** - * Method used by Json Writer to read a variable width element from - * the variable width vector and write to Json. - * - * This method should not be used externally. - * - * @param data buffer storing the variable width vector elements - * @param offset buffer storing the offsets of variable width vector elements - * @param index position of the element in the vector - * @return array of bytes - */ - public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { - final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); - final int dataLength = - offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; - final byte[] result = new byte[dataLength]; - data.getBytes(currentStartOffset, result, 0, dataLength); - return result; - } - - /** - * Method used by Json Reader to explicitly set the offsets of the variable - * width vector data. The method takes care of allocating the memory for - * offsets if the caller hasn't done so. - * - * This method should not be used externally. - * - * @param buffer ArrowBuf to store offsets for variable width elements - * @param allocator memory allocator - * @param valueCount number of elements - * @param index position of the element - * @param value offset of the element - * @return buffer holding the offsets - */ - public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (buffer == null) { - buffer = allocator.buffer(valueCount * OFFSET_WIDTH); - } - buffer.setInt(index * OFFSET_WIDTH, value); - if (index == (valueCount - 1)) { - buffer.writerIndex(valueCount * OFFSET_WIDTH); - } - - return buffer; - } + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + final int startOffset = getstartOffset(index); + while (valueBuffer.capacity() < (startOffset + dataLength)) { + reallocDataBuffer(); + } + } + + /** + * Method used by Json Writer to read a variable width element from + * the variable width vector and write to Json. + * + * This method should not be used externally. + * + * @param data buffer storing the variable width vector elements + * @param offset buffer storing the offsets of variable width vector elements + * @param index position of the element in the vector + * @return array of bytes + */ + public static byte[] get(final ArrowBuf data, final ArrowBuf offset, int index) { + final int currentStartOffset = offset.getInt(index * OFFSET_WIDTH); + final int dataLength = + offset.getInt((index + 1) * OFFSET_WIDTH) - currentStartOffset; + final byte[] result = new byte[dataLength]; + data.getBytes(currentStartOffset, result, 0, dataLength); + return result; + } + + /** + * Method used by Json Reader to explicitly set the offsets of the variable + * width vector data. The method takes care of allocating the memory for + * offsets if the caller hasn't done so. + * + * This method should not be used externally. + * + * @param buffer ArrowBuf to store offsets for variable width elements + * @param allocator memory allocator + * @param valueCount number of elements + * @param index position of the element + * @param value offset of the element + * @return buffer holding the offsets + */ + public static ArrowBuf set(ArrowBuf buffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (buffer == null) { + buffer = allocator.buffer(valueCount * OFFSET_WIDTH); + } + buffer.setInt(index * OFFSET_WIDTH, value); + if (index == (valueCount - 1)) { + buffer.writerIndex(valueCount * OFFSET_WIDTH); + } + + return buffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 1b96782681f..fc0ab3e5915 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 69325b5ea06..cc20beda3da 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.schema.ArrowFieldNode; /** * Helper class for performing generic operations on a bit vector buffer. @@ -27,139 +28,174 @@ */ public class BitVectorHelper { - /** - * Get the index of byte corresponding to bit index in validity buffer - */ - public static int byteIndex(int absoluteBitIndex) { - return absoluteBitIndex >> 3; - } - - /** - * Get the relative index of bit within the byte in validity buffer - */ - public static int bitIndex(int absoluteBitIndex) { - return absoluteBitIndex & 7; - } - - /** - * Set the bit at provided index to 1. - * - * @param validityBuffer - * @param index - */ - public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { - final int byteIndex = byteIndex(index); - final int bitIndex = bitIndex(index); - byte currentByte = validityBuffer.getByte(byteIndex); - final byte bitMask = (byte) (1L << bitIndex); + /** + * Get the index of byte corresponding to bit index in validity buffer + */ + public static int byteIndex(int absoluteBitIndex) { + return absoluteBitIndex >> 3; + } + + /** + * Get the relative index of bit within the byte in validity buffer + */ + public static int bitIndex(int absoluteBitIndex) { + return absoluteBitIndex & 7; + } + + /** + * Set the bit at provided index to 1. + * + * @param validityBuffer + * @param index + */ + public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + currentByte |= bitMask; + validityBuffer.setByte(byteIndex, currentByte); + } + + /** + * Set the bit at a given index to provided value (1 or 0) + * + * @param validityBuffer + * @param index + * @param value + */ + public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { + final int byteIndex = byteIndex(index); + final int bitIndex = bitIndex(index); + byte currentByte = validityBuffer.getByte(byteIndex); + final byte bitMask = (byte) (1L << bitIndex); + if (value != 0) { currentByte |= bitMask; - validityBuffer.setByte(byteIndex, currentByte); - } - - /** - * Set the bit at a given index to provided value (1 or 0) - * - * @param validityBuffer - * @param index - * @param value - */ - public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { - final int byteIndex = byteIndex(index); - final int bitIndex = bitIndex(index); - byte currentByte = validityBuffer.getByte(byteIndex); - final byte bitMask = (byte) (1L << bitIndex); - if (value != 0) { - currentByte |= bitMask; - } else { - currentByte -= (bitMask & currentByte); - } - validityBuffer.setByte(byteIndex, currentByte); - } - - /** - * Set the bit at a given index to provided value (1 or 0). Internally - * takes care of allocating the buffer if the caller didn't do so. - * - * @param validityBuffer - * @param allocator - * @param valueCount - * @param index - * @param value - * @return ArrowBuf - */ - public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, - int valueCount, int index, int value) { - if (validityBuffer == null) { - validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); - } - setValidityBit(validityBuffer, index, value); - if (index == (valueCount - 1)) { - validityBuffer.writerIndex(getValidityBufferSize(valueCount)); - } + } else { + currentByte -= (bitMask & currentByte); + } + validityBuffer.setByte(byteIndex, currentByte); + } + + /** + * Set the bit at a given index to provided value (1 or 0). Internally + * takes care of allocating the buffer if the caller didn't do so. + * + * @param validityBuffer + * @param allocator + * @param valueCount + * @param index + * @param value + * @return ArrowBuf + */ + public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, + int valueCount, int index, int value) { + if (validityBuffer == null) { + validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + } + setValidityBit(validityBuffer, index, value); + if (index == (valueCount - 1)) { + validityBuffer.writerIndex(getValidityBufferSize(valueCount)); + } + + return validityBuffer; + } + + /** + * Check if a bit at a given index is set or not. + * + * @param buffer + * @param index + * @return 1 if bit is set, 0 otherwise. + */ + public static int get(final ArrowBuf buffer, int index) { + final int byteIndex = index >> 3; + final byte b = buffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Compute the size of validity buffer required to manage a given number + * of elements in a vector. + * + * @param valueCount + * @return buffer size + */ + public static int getValidityBufferSize(int valueCount) { + return ((int) Math.ceil(valueCount / 8.0)); + } - return validityBuffer; - } - - /** - * Check if a bit at a given index is set or not. - * - * @param buffer - * @param index - * @return 1 if bit is set, 0 otherwise. - */ - public static int get(final ArrowBuf buffer, int index) { - final int byteIndex = index >> 3; - final byte b = buffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Compute the size of validity buffer required to manage a given number - * of elements in a vector. - * - * @param valueCount - * @return buffer size - */ - public static int getValidityBufferSize(int valueCount) { - return ((int) Math.ceil(valueCount / 8.0)); - } - - /** - * Given a validity buffer, find the number of bits that are not set. - * This is used to compute the number of null elements in a nullable vector. - * - * @param validityBuffer - * @param valueCount - * @return number of bits not set. - */ - public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { - if (valueCount == 0) { return 0; } - int count = 0; - final int sizeInBytes = getValidityBufferSize(valueCount); - - for (int i = 0; i < sizeInBytes; ++i) { - final byte byteValue = validityBuffer.getByte(i); + /** + * Given a validity buffer, find the number of bits that are not set. + * This is used to compute the number of null elements in a nullable vector. + * + * @param validityBuffer + * @param valueCount + * @return number of bits not set. + */ + public static int getNullCount(final ArrowBuf validityBuffer, final int valueCount) { + if (valueCount == 0) { + return 0; + } + int count = 0; + final int sizeInBytes = getValidityBufferSize(valueCount); + + for (int i = 0; i < sizeInBytes; ++i) { + final byte byteValue = validityBuffer.getByte(i); /* Java uses two's complement binary representation, hence 11111111_b which is -1 * when converted to Int will have 32bits set to 1. Masking the MSB and then * adding it back solves the issue. */ - count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); - } - int nullCount = (sizeInBytes * 8) - count; + count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); + } + int nullCount = (sizeInBytes * 8) - count; /* if the valueCount is not a multiple of 8, * the bits on the right were counted as null bits. */ - int remainder = valueCount % 8; - nullCount -= remainder == 0 ? 0 : 8 - remainder; - return nullCount; - } + int remainder = valueCount % 8; + nullCount -= remainder == 0 ? 0 : 8 - remainder; + return nullCount; + } + + public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { + return (byte) ((data.getByte(index) & 0xFF) >>> offset); + } + + public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { + return (byte) ((data.getByte(index) << (8 - offset))); + } - public static byte getBitsFromCurrentByte(final ArrowBuf data, final int index, final int offset) { - return (byte)((data.getByte(index) & 0xFF) >>> offset); - } + public static ArrowBuf loadValidityBuffer(final ArrowFieldNode fieldNode, + final ArrowBuf sourceValidityBuffer, + final BufferAllocator allocator) { + final int valueCount = fieldNode.getLength(); + ArrowBuf newBuffer = null; + /* either all NULLs or all non-NULLs */ + if (fieldNode.getNullCount() == 0 || fieldNode.getNullCount() == valueCount) { + newBuffer = allocator.buffer(getValidityBufferSize(valueCount)); + newBuffer.setZero(0, newBuffer.capacity()); + if (fieldNode.getNullCount() != 0) { + /* all NULLs */ + return newBuffer; + } + /* all non-NULLs */ + int fullBytesCount = valueCount / 8; + for (int i = 0; i < fullBytesCount; ++i) { + newBuffer.setByte(i, 0xFF); + } + int remainder = valueCount % 8; + if (remainder > 0) { + byte bitMask = (byte) (0xFFL >>> ((8 - remainder) & 7)); + newBuffer.setByte(fullBytesCount, bitMask); + } + } else { + /* mixed byte pattern -- create another ArrowBuf associated with the + * target allocator + */ + newBuffer = sourceValidityBuffer.retain(allocator); + } - public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte)((data.getByte(index) << (8 - offset))); - } + return newBuffer; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java index 77d214bded6..4be5d44d090 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -1,5 +1,4 @@ -/******************************************************************************* - * +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java index ae896fb7907..eca6592c54a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBigIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,337 +34,335 @@ * maintained to track which elements in the vector are null. */ public class NullableBigIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableBigIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), - allocator); - } - - /** - * Instantiate a NullableBigIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new BigIntReaderImpl(NullableBigIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.BIGINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIGINT.getType()), + allocator); + } + + /** + * Instantiate a NullableBigIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBigIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new BigIntReaderImpl(NullableBigIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIGINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBigIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableBigIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, BigIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBigIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBigIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBigIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, BigIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, BigIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BigIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBigIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BigIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BigIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBigIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableBigIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableBigIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableBigIntVector to){ - this.to = to; - } - - @Override - public NullableBigIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBigIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableBigIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableBigIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBigIntVector to) { + this.to = to; + } + + @Override + public NullableBigIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBigIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java index dda4c434876..bd363f5b7f1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableBitVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector; @@ -39,150 +38,160 @@ * to a single bit in the underlying data stream backing the vector. */ public class NullableBitVector extends BaseNullableFixedWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.BIT.getType()), - allocator); - } - - /** - * Instantiate a NullableBitVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, (byte)0); - reader = new BitReaderImpl(NullableBitVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.BIT; - } - - /** - * Sets the desired value capacity for the vector. This function doesn't - * allocate any memory for the vector. - * @param valueCount desired number of elements in the vector - */ - @Override - public void setInitialCapacity(int valueCount) { - final int size = getValidityBufferSizeFromCount(valueCount); - if (size > MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); - } - valueAllocationSizeInBytes = size; - validityAllocationSizeInBytes = size; - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity(){ - return (int)(validityBuffer.capacity() * 8L); - } - - /** - * Get the potential buffer size for a particular number of records. - * @param count desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int count) { - if (count == 0) { return 0; } - return 2 * getValidityBufferSizeFromCount(count); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - return getBufferSizeFor(valueCount); - } - - /** - * Slice this vector at desired index and length and transfer the - * corresponding data to the target vector. - * @param startIndex start position of the split in source vector. - * @param length length of the split. - * @param target destination vector - */ - public void splitAndTransferTo(int startIndex, int length, - BaseNullableFixedWidthVector target) { - compareTypes(target, "splitAndTransferTo"); - target.clear(); - target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, - validityBuffer, target.validityBuffer); - target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, - valueBuffer, target.valueBuffer); - - target.setValueCount(length); - } - - private ArrowBuf splitAndTransferBuffer(int startIndex, int length, - BaseNullableFixedWidthVector target, - ArrowBuf sourceBuffer, ArrowBuf destBuffer) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = getValidityBufferSizeFromCount(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { + private final FieldReader reader; + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.BIT.getType()), + allocator); + } + + /** + * Instantiate a NullableBitVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableBitVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, (byte) 0); + reader = new BitReaderImpl(NullableBitVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.BIT; + } + + /** + * Sets the desired value capacity for the vector. This function doesn't + * allocate any memory for the vector. + * + * @param valueCount desired number of elements in the vector + */ + @Override + public void setInitialCapacity(int valueCount) { + final int size = getValidityBufferSizeFromCount(valueCount); + if (size > MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); + } + valueAllocationSizeInBytes = size; + validityAllocationSizeInBytes = size; + } + + /** + * Get the current value capacity for the vector + * + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * Get the potential buffer size for a particular number of records. + * + * @param count desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int count) { + if (count == 0) { + return 0; + } + return 2 * getValidityBufferSizeFromCount(count); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + return getBufferSizeFor(valueCount); + } + + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * + * @param startIndex start position of the split in source vector. + * @param length length of the split. + * @param target destination vector + */ + public void splitAndTransferTo(int startIndex, int length, + BaseNullableFixedWidthVector target) { + compareTypes(target, "splitAndTransferTo"); + target.clear(); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, + validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, + valueBuffer, target.valueBuffer); + + target.setValueCount(length); + } + + private ArrowBuf splitAndTransferBuffer(int startIndex, int length, + BaseNullableFixedWidthVector target, + ArrowBuf sourceBuffer, ArrowBuf destBuffer) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = getValidityBufferSizeFromCount(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { /* slice */ - if (destBuffer != null) { - destBuffer.release(); - } - destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); - destBuffer.retain(1); - } - else { + if (destBuffer != null) { + destBuffer.release(); + } + destBuffer = sourceBuffer.slice(firstByteSource, byteSizeTarget); + destBuffer.retain(1); + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. * Each byte in the target is composed by a part in i-th byte, * another part in (i+1)-th byte. */ - destBuffer = allocator.buffer(byteSizeTarget); - destBuffer.readerIndex(0); - destBuffer.setZero(0, destBuffer.capacity()); + destBuffer = allocator.buffer(byteSizeTarget); + destBuffer.readerIndex(0); + destBuffer.setZero(0, destBuffer.capacity()); - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, firstByteSource + i + 1, offset); - destBuffer.setByte(i, (b1 + b2)); - } + destBuffer.setByte(i, (b1 + b2)); + } /* Copying the last piece is done in the following manner: * if the source vector has 1 or more bytes remaining, we copy @@ -193,312 +202,315 @@ private ArrowBuf splitAndTransferBuffer(int startIndex, int length, * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, - firstByteSource + byteSizeTarget, offset); - - destBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, - firstByteSource + byteSizeTarget - 1, offset); - destBuffer.setByte(byteSizeTarget - 1, b1); - } - } - } - - return destBuffer; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - private int getBit(int index) { - final int byteIndex = index >> 3; - final byte b = valueBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return getBit(index); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableBitHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = getBit(index); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Boolean getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return new Boolean (getBit(index) != 0); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableBitVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); - } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); - } + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(sourceBuffer, + firstByteSource + byteSizeTarget, offset); + + destBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(sourceBuffer, + firstByteSource + byteSizeTarget - 1, offset); + destBuffer.setByte(byteSizeTarget - 1, b1); + } } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, BitHolder holder) { + } + + return destBuffer; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + private int getBit(int index) { + final int byteIndex = index >> 3; + final byte b = valueBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return getBit(index); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableBitHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = getBit(index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Boolean getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return new Boolean(getBit(index) != 0); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableBitVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableBitVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableBitVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableBitHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); if (holder.value != 0) { - BitVectorHelper.setValidityBitToOne(valueBuffer, index); + BitVectorHelper.setValidityBitToOne(valueBuffer, index); } else { - BitVectorHelper.setValidityBit(valueBuffer, index, 0); + BitVectorHelper.setValidityBit(valueBuffer, index, 0); } - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableBitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, BitHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, BitHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, BitHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + if (holder.value != 0) { + BitVectorHelper.setValidityBitToOne(valueBuffer, index); + } else { + BitVectorHelper.setValidityBit(valueBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableBitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableBitHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, BitHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, BitHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableBitVector)to); - } - - private class TransferImpl implements TransferPair { - NullableBitVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableBitVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableBitVector to){ - this.to = to; - } - - @Override - public NullableBitVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableBitVector) to); + } + + private class TransferImpl implements TransferPair { + NullableBitVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableBitVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableBitVector to) { + this.to = to; + } + + @Override + public NullableBitVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableBitVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java index 566cb2729d4..c5ddc9e87be 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateDayVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,338 +35,336 @@ * maintained to track which elements in the vector are null. */ public class NullableDateDayVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDateDayVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), - allocator); - } - - /** - * Instantiate a NullableDateDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateDayReaderImpl(NullableDateDayVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DATEDAY; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableDateDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateDayReaderImpl(NullableDateDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateDayHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDateDayHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getInt(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DateDayHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateDayVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DateDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DateDayHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateDayHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateDayHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateDayVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDateDayVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDateDayVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableDateDayVector to){ - this.to = to; - } - - @Override - public NullableDateDayVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateDayVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDateDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDateDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateDayVector to) { + this.to = to; + } + + @Override + public NullableDateDayVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateDayVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java index 0f439fde746..7e37055dab1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDateMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,341 +37,339 @@ * maintained to track which elements in the vector are null. */ public class NullableDateMilliVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDateMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new DateMilliReaderImpl(NullableDateMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DATEMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDateMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); - final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, - org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DateMilliHolder holder){ + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.DATEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableDateMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDateMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DateMilliReaderImpl(NullableDateMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DATEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDateMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDateMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDateMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDateMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DateMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DateMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DateMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDateMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DateMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DateMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDateMilliVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDateMilliVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableDateMilliVector to){ - this.to = to; - } - - @Override - public NullableDateMilliVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDateMilliVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDateMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDateMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableDateMilliVector to) { + this.to = to; + } + + @Override + public NullableDateMilliVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDateMilliVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java index 20ee3800955..dcc551094ae 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableDecimalVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,391 +37,391 @@ * maintained to track which elements in the vector are null. */ public class NullableDecimalVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 16; - private final FieldReader reader; - - private final int precision; - private final int scale; - - /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableDecimalVector(String name, BufferAllocator allocator, - int precision, int scale) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), - allocator); - } - - /** - * Instantiate a NullableDecimalVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal)fieldType.getType(); - reader = new DecimalReaderImpl(NullableDecimalVector.this); - this.precision = arrowType.getPrecision(); - this.scale = arrowType.getScale(); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.DECIMAL; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public ArrowBuf get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableDecimalHolder holder) { - if (isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.buffer = valueBuffer; - holder.precision = precision; - holder.scale = scale; - holder.start = index * TYPE_WIDTH; - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public BigDecimal getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { - if (from.isSet(fromIndex) != 0) { - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, - thisIndex * TYPE_WIDTH, TYPE_WIDTH); - BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - /** - * Return scale for the decimal value - */ - public int getScale() { return scale; } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param buffer ArrowBuf containing decimal value. - */ - public void set(int index, ArrowBuf buffer) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param start start index of data in the buffer - * @param buffer ArrowBuf containing decimal value. - */ - public void set(int index, int start, ArrowBuf buffer) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value BigDecimal containing decimal value. - */ - public void set(int index, BigDecimal value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - DecimalUtility.checkPrecisionAndScale(value, precision, scale); - DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, DecimalHolder holder){ + public static final byte TYPE_WIDTH = 16; + private final FieldReader reader; + + private final int precision; + private final int scale; + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, BufferAllocator allocator, + int precision, int scale) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)), + allocator); + } + + /** + * Instantiate a NullableDecimalVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + org.apache.arrow.vector.types.pojo.ArrowType.Decimal arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Decimal) fieldType.getType(); + reader = new DecimalReaderImpl(NullableDecimalVector.this); + this.precision = arrowType.getPrecision(); + this.scale = arrowType.getScale(); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.DECIMAL; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDecimalHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.buffer = valueBuffer; + holder.precision = precision; + holder.scale = scale; + holder.start = index * TYPE_WIDTH; + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public BigDecimal getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableDecimalVector from) { + if (from.isSet(fromIndex) != 0) { + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableDecimalVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableDecimalVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + /** + * Return scale for the decimal value + */ + public int getScale() { + return scale; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void set(int index, int start, ArrowBuf buffer) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, buffer, start, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void set(int index, BigDecimal value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + DecimalUtility.checkPrecisionAndScale(value, precision, scale); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); - } - - /** - * Same as {@link #set(int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param buffer ArrowBuf containing decimal value. - */ - public void setSafe(int index, ArrowBuf buffer) { - handleSafe(index); - set(index, buffer); - } - - /** - * Same as {@link #set(int, int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param start start index of data in the buffer - * @param buffer ArrowBuf containing decimal value. - */ - public void setSafe(int index, int start, ArrowBuf buffer) { - handleSafe(index); - set(index, start, buffer); - } - - /** - * Same as {@link #set(int, BigDecimal)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value BigDecimal containing decimal value. - */ - public void setSafe(int index, BigDecimal value){ - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, DecimalHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, DecimalHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DecimalHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, ArrowBuf buffer) { + handleSafe(index); + set(index, buffer); + } + + /** + * Same as {@link #set(int, int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param start start index of data in the buffer + * @param buffer ArrowBuf containing decimal value. + */ + public void setSafe(int index, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, start, buffer); + } + + /** + * Same as {@link #set(int, BigDecimal)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value BigDecimal containing decimal value. + */ + public void setSafe(int index, BigDecimal value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DecimalHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DecimalHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void set(int index, int isSet, int start, ArrowBuf buffer) { + if (isSet > 0) { + set(index, start, buffer); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of the value in the buffer - * @param buffer buffer containing the value to be stored in the vector - */ - public void set(int index, int isSet, int start, ArrowBuf buffer) { - if (isSet > 0) { - set(index, start, buffer); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles - * the case when the position of new value is beyond the current value - * capacity of the vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param start start position of the value in the buffer - * @param buffer buffer containing the value to be stored in the vector - */ - public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { - handleSafe(index); - set(index, isSet, start, buffer); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableDecimalVector)to); - } - - private class TransferImpl implements TransferPair { - NullableDecimalVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, - NullableDecimalVector.this.scale); - } - - public TransferImpl(NullableDecimalVector to){ - this.to = to; - } - - @Override - public NullableDecimalVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); - } - } + } + } + + /** + * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles + * the case when the position of new value is beyond the current value + * capacity of the vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param start start position of the value in the buffer + * @param buffer buffer containing the value to be stored in the vector + */ + public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { + handleSafe(index); + set(index, isSet, start, buffer); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableDecimalVector) to); + } + + private class TransferImpl implements TransferPair { + NullableDecimalVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableDecimalVector(ref, allocator, NullableDecimalVector.this.precision, + NullableDecimalVector.this.scale); + } + + public TransferImpl(NullableDecimalVector to) { + this.to = to; + } + + @Override + public NullableDecimalVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableDecimalVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java index eb895a29beb..834f16c211c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat4Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,338 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat4Vector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableFloat4Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), - allocator); - } - - /** - * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float4ReaderImpl(NullableFloat4Vector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.FLOAT4; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public float get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT4.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat4Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float4ReaderImpl(NullableFloat4Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public float get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat4Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Float getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getFloat(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableFloat4Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getFloat(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Float getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getFloat(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, float value) { - valueBuffer.setFloat(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, float value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, Float4Holder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat4Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, float value) { + valueBuffer.setFloat(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, float value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, float)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, float value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, Float4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, Float4Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float4Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, float)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, float value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float4Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, float value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, float value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, float)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, float value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static float get(final ArrowBuf buffer, final int index) { - return buffer.getFloat(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat4Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableFloat4Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableFloat4Vector to){ - this.to = to; - } - - @Override - public NullableFloat4Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, float)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, float value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static float get(final ArrowBuf buffer, final int index) { + return buffer.getFloat(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat4Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableFloat4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableFloat4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat4Vector to) { + this.to = to; + } + + @Override + public NullableFloat4Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat4Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java index 51aac5ffead..b054d9b05c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableFloat8Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,338 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableFloat8Vector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableFloat8Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), - allocator); - } - - /** - * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new Float8ReaderImpl(NullableFloat8Vector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.FLOAT8; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public double get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.FLOAT8.getType()), + allocator); + } + + /** + * Instantiate a NullableFloat8Vector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableFloat8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new Float8ReaderImpl(NullableFloat8Vector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.FLOAT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public double get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableFloat8Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Double getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getDouble(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableFloat8Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getDouble(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Double getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getDouble(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, double value) { - valueBuffer.setDouble(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, double value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, Float8Holder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableFloat8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableFloat8Vector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableFloat8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, double value) { + valueBuffer.setDouble(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, double value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, double)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, double value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, Float8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, Float8Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, Float8Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, double)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, double value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableFloat8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableFloat8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, Float8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, Float8Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, double value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, double value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, double)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, double value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static double get(final ArrowBuf buffer, final int index) { - return buffer.getDouble(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableFloat8Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableFloat8Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableFloat8Vector to){ - this.to = to; - } - - @Override - public NullableFloat8Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, double)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, double value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static double get(final ArrowBuf buffer, final int index) { + return buffer.getDouble(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableFloat8Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableFloat8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableFloat8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableFloat8Vector to) { + this.to = to; + } + + @Override + public NullableFloat8Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableFloat8Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java index 1fadbf23dce..e5dfbcedcf9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,338 +34,346 @@ * maintained to track which elements in the vector are null. */ public class NullableIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), - allocator); - } - - /** - * Instantiate a NullableIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntReaderImpl(NullableIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.INT.getType()), + allocator); + } + + /** + * Instantiate a NullableIntVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntReaderImpl(NullableIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getInt(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntVector to){ - this.to = to; - } - - @Override - public NullableIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + *

+ * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntVector to) { + this.to = to; + } + + @Override + public NullableIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java index 8524905cf37..8d2fb5661d9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalDayVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,396 +36,394 @@ * vector are null. */ public class NullableIntervalDayVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private static final byte MILLISECOND_OFFSET = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntervalDayVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), - allocator); - } - - /** - * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INTERVALDAY; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public ArrowBuf get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - return null; - } - return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntervalDayHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } + private static final byte TYPE_WIDTH = 8; + private static final byte MILLISECOND_OFFSET = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALDAY.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalDayVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalDayVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalDayReaderImpl(NullableIntervalDayVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALDAY; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalDayHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + final int startIndex = index * TYPE_WIDTH; + holder.isSet = 1; + holder.days = valueBuffer.getInt(startIndex); + holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { final int startIndex = index * TYPE_WIDTH; - holder.isSet = 1; - holder.days = valueBuffer.getInt(startIndex); - holder.milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Period getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final int startIndex = index * TYPE_WIDTH; - final int days = valueBuffer.getInt(startIndex); - final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - final Period p = new Period(); - return p.plusDays(days).plusMillis(milliseconds); - } - } - - /** - * Get the Interval value at a given index as a {@link StringBuilder} object - * @param index position of the element - * @return String Builder object with Interval value as - * [days, hours, minutes, seconds, millis] - */ - public StringBuilder getAsStringBuilder(int index) { - if (isSet(index) == 0) { - return null; - }else{ - return getAsStringBuilderHelper(index); - } - } - - private StringBuilder getAsStringBuilderHelper(int index) { - final int startIndex = index * TYPE_WIDTH; - - final int days = valueBuffer.getInt(startIndex); - int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - - final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - - final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - - final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - - final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - - return(new StringBuilder(). - append(days).append(dayString). - append(hours).append(":"). - append(minutes).append(":"). - append(seconds).append("."). - append(millis)); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { - if (from.isSet(fromIndex) != 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); - from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, - thisIndex * TYPE_WIDTH, TYPE_WIDTH); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, ArrowBuf value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param days days for the interval - * @param milliseconds milliseconds for the interval - */ - public void set(int index, int days, int milliseconds){ - final int offsetIndex = index * TYPE_WIDTH; - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - valueBuffer.setInt(offsetIndex, days); - valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - set(index, holder.days, holder.milliseconds); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntervalDayHolder holder){ + final int days = valueBuffer.getInt(startIndex); + final int milliseconds = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + final Period p = new Period(); + return p.plusDays(days).plusMillis(milliseconds); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [days, hours, minutes, seconds, millis] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + } else { + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + final int startIndex = index * TYPE_WIDTH; + + final int days = valueBuffer.getInt(startIndex); + int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); + + final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + + final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + + final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; + + return (new StringBuilder(). + append(days).append(dayString). + append(hours).append(":"). + append(minutes).append(":"). + append(seconds).append("."). + append(millis)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + if (from.isSet(fromIndex) != 0) { + BitVectorHelper.setValidityBitToOne(validityBuffer, thisIndex); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalDayVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalDayVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void set(int index, int days, int milliseconds) { + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setInt(offsetIndex, days); + valueBuffer.setInt((offsetIndex + MILLISECOND_OFFSET), milliseconds); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { set(index, holder.days, holder.milliseconds); - } - - /** - * Same as {@link #set(int, ArrowBuf)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, ArrowBuf value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param days days for the interval - * @param milliseconds milliseconds for the interval - */ - public void setSafe(int index, int days, int milliseconds) { - handleSafe(index); - set(index, days, milliseconds); - } - - /** - * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntervalDayHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntervalDayHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalDayHolder holder) { + set(index, holder.days, holder.milliseconds); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param days days for the interval + * @param milliseconds milliseconds for the interval + */ + public void setSafe(int index, int days, int milliseconds) { + handleSafe(index); + set(index, days, milliseconds); + } + + /** + * Same as {@link #set(int, NullableIntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalDayHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalDayHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalDayHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void set(int index, int isSet, int days, int milliseconds) { + if (isSet > 0) { + set(index, days, milliseconds); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param days days component of interval - * @param milliseconds millisecond component of interval - */ - public void set(int index, int isSet, int days, int milliseconds) { - if (isSet > 0) { - set(index, days, milliseconds); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param days days component of interval - * @param milliseconds millisecond component of interval - */ - public void setSafe(int index, int isSet, int days, int milliseconds) { - handleSafe(index); - set(index, isSet, days, milliseconds); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalDayVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntervalDayVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntervalDayVector to){ - this.to = to; - } - - @Override - public NullableIntervalDayVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param days days component of interval + * @param milliseconds millisecond component of interval + */ + public void setSafe(int index, int isSet, int days, int milliseconds) { + handleSafe(index); + set(index, isSet, days, milliseconds); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalDayVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalDayVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntervalDayVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalDayVector to) { + this.to = to; + } + + @Override + public NullableIntervalDayVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalDayVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java index 6aeae899289..ac121d2f078 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableIntervalYearVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,356 +34,354 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableIntervalYearVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableIntervalYearVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), - allocator); - } - - /** - * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.INTERVALYEAR; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableIntervalYearHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Period getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final int interval = valueBuffer.getInt(index * TYPE_WIDTH); - final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final Period p = new Period(); - return p.plusYears(years).plusMonths(months); - } - } - - /** - * Get the Interval value at a given index as a {@link StringBuilder} object - * @param index position of the element - * @return String Builder object with Interval value as - * [years, months] - */ - public StringBuilder getAsStringBuilder(int index) { - if (isSet(index) == 0) { - return null; - }else{ - return getAsStringBuilderHelper(index); - } - } - - private StringBuilder getAsStringBuilderHelper(int index) { - int value = valueBuffer.getInt(index * TYPE_WIDTH); - - final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); - final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); - - final String yearString = (Math.abs(years) == 1) ? " year " : " years "; - final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - - return(new StringBuilder(). - append(years).append(yearString). - append(months).append(monthString)); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, IntervalYearHolder holder){ + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.INTERVALYEAR.getType()), + allocator); + } + + /** + * Instantiate a NullableIntervalYearVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableIntervalYearVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new IntervalYearReaderImpl(NullableIntervalYearVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.INTERVALYEAR; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableIntervalYearHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Period getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final int interval = valueBuffer.getInt(index * TYPE_WIDTH); + final int years = (interval / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (interval % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final Period p = new Period(); + return p.plusYears(years).plusMonths(months); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object + * @param index position of the element + * @return String Builder object with Interval value as + * [years, months] + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + } else { + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + int value = valueBuffer.getInt(index * TYPE_WIDTH); + + final int years = (value / org.apache.arrow.vector.util.DateUtility.yearsToMonths); + final int months = (value % org.apache.arrow.vector.util.DateUtility.yearsToMonths); + + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + + return (new StringBuilder(). + append(years).append(yearString). + append(months).append(monthString)); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableIntervalYearVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableIntervalYearVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, IntervalYearHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, IntervalYearHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, IntervalYearHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableIntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableIntervalYearHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, IntervalYearHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, IntervalYearHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableIntervalYearVector)to); - } - - private class TransferImpl implements TransferPair { - NullableIntervalYearVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableIntervalYearVector to){ - this.to = to; - } - - @Override - public NullableIntervalYearVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableIntervalYearVector) to); + } + + private class TransferImpl implements TransferPair { + NullableIntervalYearVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableIntervalYearVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableIntervalYearVector to) { + this.to = to; + } + + @Override + public NullableIntervalYearVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableIntervalYearVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java index ca57b03c032..5dd1492a3e0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableSmallIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,365 +34,363 @@ * maintained to track which elements in the vector are null. */ public class NullableSmallIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 2; - private final FieldReader reader; - - /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableSmallIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), - allocator); - } - - /** - * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new SmallIntReaderImpl(NullableSmallIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.SMALLINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public short get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.SMALLINT.getType()), + allocator); + } + + /** + * Instantiate a NullableSmallIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableSmallIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new SmallIntReaderImpl(NullableSmallIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.SMALLINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public short get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableSmallIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getShort(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Short getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getShort(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableSmallIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getShort(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Short getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getShort(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); - } - - private void setValue(int index, short value) { - valueBuffer.setShort(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, short value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, SmallIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableSmallIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableSmallIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableSmallIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + private void setValue(int index, short value) { + valueBuffer.setShort(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, short value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, short)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, short value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, SmallIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, SmallIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, SmallIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, short)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, short value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableSmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableSmallIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, SmallIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, SmallIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, short value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, short value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, short)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, short value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static short get(final ArrowBuf buffer, final int index) { - return buffer.getShort(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableSmallIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableSmallIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableSmallIntVector to){ - this.to = to; - } - - @Override - public NullableSmallIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, short)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, short value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static short get(final ArrowBuf buffer, final int index) { + return buffer.getShort(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableSmallIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableSmallIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableSmallIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableSmallIntVector to) { + this.to = to; + } + + @Override + public NullableSmallIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableSmallIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java index 92d8bec0864..25da4f68d50 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMicroVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,337 +36,335 @@ * vector are null. */ public class NullableTimeMicroVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeMicroVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMEMICRO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMicroReaderImpl(NullableTimeMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMicroHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeMicroHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeMicroHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMicroVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMicroVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeMicroHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMicroHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMicroHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMicroVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeMicroVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeMicroVector to){ - this.to = to; - } - - @Override - public NullableTimeMicroVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMicroVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMicroVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeMicroVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMicroVector to) { + this.to = to; + } + + @Override + public NullableTimeMicroVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMicroVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java index 5b7bd39f99b..7d2d5d1f10f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,338 +36,336 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeMilliVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMEMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } - org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), - org.joda.time.DateTimeZone.UTC); - return ldt; - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeMilliHolder holder){ + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMEMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeMilliReaderImpl(NullableTimeMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMEMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } + org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), + org.joda.time.DateTimeZone.UTC); + return ldt; + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeMilliVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeMilliVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeMilliVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeMilliVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeMilliVector to){ - this.to = to; - } - - @Override - public NullableTimeMilliVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeMilliVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeMilliVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeMilliVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeMilliVector to) { + this.to = to; + } + + @Override + public NullableTimeMilliVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeMilliVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java index 100e4fd0711..e93a63fe2a1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeNanoVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,337 +34,335 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeNanoVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeNanoVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMENANO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMENANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeNanoReaderImpl(NullableTimeNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMENANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeNanoHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeNanoHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeNanoHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeNanoVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeNanoVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeNanoHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeNanoHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeNanoHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer,final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeNanoVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeNanoVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeNanoVector to){ - this.to = to; - } - - @Override - public NullableTimeNanoVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeNanoVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeNanoVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeNanoVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeNanoVector to) { + this.to = to; + } + + @Override + public NullableTimeNanoVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeNanoVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java index de512db3591..8329fa78139 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeSecVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,338 +34,336 @@ * maintained to track which elements in the vector are null. */ public class NullableTimeSecVector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeSecVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TimeSecReaderImpl(NullableTimeSecVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESEC; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TimeSecReaderImpl(NullableTimeSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeSecHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeSecHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getInt(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeSecHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeSecVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTimeSecVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeSecVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeSecHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeSecHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeSecHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, int)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static int get(final ArrowBuf buffer, final int index) { - return buffer.getInt(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeSecVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTimeSecVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTimeSecVector to){ - this.to = to; - } - - @Override - public NullableTimeSecVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, int)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int get(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeSecVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTimeSecVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTimeSecVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTimeSecVector to) { + this.to = to; + } + + @Override + public NullableTimeSecVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeSecVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java index 6b53deb6b6b..c7ad141549b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMicroTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMICROTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMicroTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMicroTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMicroTZReaderImpl(NullableTimeStampMicroTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICROTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMicroTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroTZVector to = new NullableTimeStampMicroTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java index 9debe1daad3..5a34b194ce7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMicroVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,188 +34,186 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMicroVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMICRO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMicroHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMICRO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMicroVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMicroVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMicroReaderImpl(NullableTimeStampMicroVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMICRO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMicroHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { /* value is truncated when converting microseconds to milliseconds in order to use DateTime type */ - final long micros = valueBuffer.getLong(index * TYPE_WIDTH); - final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, - org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMicroHolder holder){ + final long micros = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(micros); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMicroHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMicroVector )to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMicroHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMicroHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMicroHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMicroHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMicroVector to = new NullableTimeStampMicroVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMicroVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java index 2bb7c003a0a..f9736aed7be 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,185 +34,183 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMilliTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMILLITZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMilliTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMilliTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MILLISECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampMilliTZReaderImpl(NullableTimeStampMilliTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLITZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMilliTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliTZVector to = new NullableTimeStampMilliTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java index 8f4bf4acb99..fc38a6bd507 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampMilliVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampMilliVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPMILLI; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampMilliHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long millis = valueBuffer.getLong(index * TYPE_WIDTH); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, - org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampMilliHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPMILLI.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampMilliVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampMilliVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampMilliReaderImpl(NullableTimeStampMilliVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPMILLI; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampMilliHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long millis = valueBuffer.getLong(index * TYPE_WIDTH); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampMilliHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampMilliVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampMilliHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampMilliHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampMilliHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampMilliHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampMilliVector to = new NullableTimeStampMilliVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampMilliVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java index 47fb033aae6..8eb48462c99 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,186 +34,184 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampNanoTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPNANOTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampNanoTZHolder holder) { - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampNanoTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.NANOSECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampNanoTZReaderImpl(NullableTimeStampNanoTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANOTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampNanoTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoTZVector to = new NullableTimeStampNanoTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java index d464873e8d2..6cc5c905783 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampNanoVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,187 +34,185 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampNanoVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPNANO; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampNanoHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); - final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, - org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampNanoHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPNANO.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampNanoVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampNanoVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampNanoReaderImpl(NullableTimeStampNanoVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPNANO; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampNanoHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long nanos = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(nanos); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampNanoHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampNanoVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampNanoHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampNanoHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampNanoHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampNanoHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampNanoVector to = new NullableTimeStampNanoVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampNanoVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java index 3096f8c558b..0593ac76dce 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecTZVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,185 +33,183 @@ * (bit vector) is maintained to track which elements in the vector are null. */ public class NullableTimeStampSecTZVector extends NullableTimeStampVector { - private final FieldReader reader; - private final String timeZone; - - /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { - this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), - allocator); - } - - /** - * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp)fieldType.getType(); - timeZone = arrowType.getTimezone(); - reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPSECTZ; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampSecTZHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampSecTZHolder holder){ + private final FieldReader reader; + private final String timeZone; + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, BufferAllocator allocator, String timeZone) { + this(name, FieldType.nullable(new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, timeZone)), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecTZVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecTZVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + org.apache.arrow.vector.types.pojo.ArrowType.Timestamp arrowType = (org.apache.arrow.vector.types.pojo.ArrowType.Timestamp) fieldType.getType(); + timeZone = arrowType.getTimezone(); + reader = new TimeStampSecTZReaderImpl(NullableTimeStampSecTZVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSECTZ; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecTZHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return valueBuffer.getLong(index * TYPE_WIDTH); + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampSecTZHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecTZVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecTZHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecTZHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecTZHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecTZHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecTZVector to = new NullableTimeStampSecTZVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecTZVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java index 1c281ef492f..1afdb30d045 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampSecVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,187 +34,185 @@ * maintained to track which elements in the vector are null. */ public class NullableTimeStampSecVector extends NullableTimeStampVector { - private final FieldReader reader; - - /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), - allocator); - } - - /** - * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, fieldType, allocator); - reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader() { - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TIMESTAMPSEC; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTimeStampSecHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public LocalDateTime getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - final long secs = valueBuffer.getLong(index * TYPE_WIDTH); - final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); - final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, - org.joda.time.DateTimeZone.UTC); - return localDateTime; - } - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TimeStampSecHolder holder){ + private final FieldReader reader; + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TIMESTAMPSEC.getType()), + allocator); + } + + /** + * Instantiate a NullableTimeStampSecVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampSecVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, fieldType, allocator); + reader = new TimeStampSecReaderImpl(NullableTimeStampSecVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TIMESTAMPSEC; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTimeStampSecHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public LocalDateTime getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long secs = valueBuffer.getLong(index * TYPE_WIDTH); + final long millis = java.util.concurrent.TimeUnit.SECONDS.toMillis(secs); + final org.joda.time.LocalDateTime localDateTime = new org.joda.time.LocalDateTime(millis, + org.joda.time.DateTimeZone.UTC); + return localDateTime; + } + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TimeStampSecHolder holder){ - handleSafe(index); - set(index, holder); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, - field.getFieldType(), allocator); - return new TransferImpl(to); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTimeStampSecVector)to); - } + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TimeStampSecHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, NullableTimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTimeStampSecHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TimeStampSecHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TimeStampSecHolder holder) { + handleSafe(index); + set(index, holder); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + NullableTimeStampSecVector to = new NullableTimeStampSecVector(ref, + field.getFieldType(), allocator); + return new TransferImpl(to); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTimeStampSecVector) to); + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java index 9a8771ae7bb..d65cb83e151 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTimeStampVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,191 +29,191 @@ * maintained to track which elements in the vector are null. */ public abstract class NullableTimeStampVector extends BaseNullableFixedWidthVector { - protected static final byte TYPE_WIDTH = 8; - - /** - * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - protected void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + protected static final byte TYPE_WIDTH = 8; + + /** + * Instantiate a NullableTimeStampVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTimeStampVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTimeStampVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFromSafe(int, int, NullableTimeStampVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTimeStampVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + protected void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, long)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static long get(final ArrowBuf buffer, final int index) { - return buffer.getLong(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - public class TransferImpl implements TransferPair { - NullableTimeStampVector to; - - public TransferImpl(NullableTimeStampVector to) { - this.to = to; - } - - @Override - public NullableTimeStampVector getTo(){ - return to; - } - - @Override - public void transfer() { - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + public class TransferImpl implements TransferPair { + NullableTimeStampVector to; + + public TransferImpl(NullableTimeStampVector to) { + this.to = to; + } + + @Override + public NullableTimeStampVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTimeStampVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java index 578c1e18594..106359ddd96 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableTinyIntVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,365 +34,363 @@ * maintained to track which elements in the vector are null. */ public class NullableTinyIntVector extends BaseNullableFixedWidthVector { - public static final byte TYPE_WIDTH = 1; - private final FieldReader reader; - - /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableTinyIntVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), - allocator); - } - - /** - * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new TinyIntReaderImpl(NullableTinyIntVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.TINYINT; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public byte get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + public static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.TINYINT.getType()), + allocator); + } + + /** + * Instantiate a NullableTinyIntVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableTinyIntVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new TinyIntReaderImpl(NullableTinyIntVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.TINYINT; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableTinyIntHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableTinyIntHolder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Byte getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getByte(index * TYPE_WIDTH); - } - } - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - /** - * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - private void setValue(int index, int value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - private void setValue(int index, byte value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, byte value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, TinyIntHolder holder){ + } + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableTinyIntVector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + /** + * Same as {@link #copyFrom(int, int, NullableTinyIntVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableTinyIntVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, byte)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, byte value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, TinyIntHolder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, TinyIntHolder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, TinyIntHolder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableTinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableTinyIntHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, TinyIntHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, TinyIntHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - /** - * Store the given value at a particular position in the vector. isSet indicates - * whether the value is NULL or not. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void set(int index, int isSet, byte value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Same as {@link #set(int, int, byte)} except that it handles the case - * when index is greater than or equal to current value capacity of the - * vector. - * @param index position of the new value - * @param isSet 0 for NULL value, 1 otherwise - * @param value element value - */ - public void setSafe(int index, int isSet, byte value) { - handleSafe(index); - set(index, isSet, value); - } - - /** - * Given a data buffer, get the value stored at a particular position - * in the vector. - * - * This method should not be used externally. - * - * @param buffer data buffer - * @param index position of the element. - * @return value stored at the index. - */ - public static byte get(final ArrowBuf buffer, final int index) { - return buffer.getByte(index * TYPE_WIDTH); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableTinyIntVector)to); - } - - private class TransferImpl implements TransferPair { - NullableTinyIntVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableTinyIntVector to){ - this.to = to; - } - - @Override - public NullableTinyIntVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); - } - } + } + } + + /** + * Same as {@link #set(int, int, byte)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value element value + */ + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + * This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static byte get(final ArrowBuf buffer, final int index) { + return buffer.getByte(index * TYPE_WIDTH); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableTinyIntVector) to); + } + + private class TransferImpl implements TransferPair { + NullableTinyIntVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableTinyIntVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableTinyIntVector to) { + this.to = to; + } + + @Override + public NullableTinyIntVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableTinyIntVector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java index ba88b96c7c2..52f6c151bdf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt1Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,288 +33,286 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt1Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 1; - private final FieldReader reader; - - public NullableUInt1Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), - allocator); - } - - public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt1ReaderImpl(NullableUInt1Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT1; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public byte get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 1; + private final FieldReader reader; + + public NullableUInt1Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT1.getType()), + allocator); + } + + public NullableUInt1Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt1ReaderImpl(NullableUInt1Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT1; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public byte get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt1Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getByte(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Byte getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt1Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getByte(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Byte getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getByte(index * TYPE_WIDTH); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - private void setValue(int index, byte value) { - valueBuffer.setByte(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, byte value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt1Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt1Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt1Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + private void setValue(int index, byte value) { + valueBuffer.setByte(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, byte value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, byte)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, byte value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt1Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt1Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt1Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, byte)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, byte value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt1Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt1Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt1Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, byte value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, byte value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, byte value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt1Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt1Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt1Vector to){ - this.to = to; - } - - @Override - public NullableUInt1Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, byte value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt1Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt1Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt1Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt1Vector to) { + this.to = to; + } + + @Override + public NullableUInt1Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt1Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java index 9a097a60c48..42ba6e139bf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt2Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,288 +34,286 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt2Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 2; - private final FieldReader reader; - - public NullableUInt2Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), - allocator); - } - - public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt2ReaderImpl(NullableUInt2Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT2; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public char get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 2; + private final FieldReader reader; + + public NullableUInt2Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT2.getType()), + allocator); + } + + public NullableUInt2Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt2ReaderImpl(NullableUInt2Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT2; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public char get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt2Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getChar(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Character getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getChar(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt2Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getChar(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Character getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getChar(index * TYPE_WIDTH); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); - } - - private void setValue(int index, char value) { - valueBuffer.setChar(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, char value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt2Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt2Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt2Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + private void setValue(int index, char value) { + valueBuffer.setChar(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, char value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, char)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, char value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt2Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt2Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt2Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, char)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, char value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt2Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt2Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt2Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, char value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, char value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, char value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt2Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt2Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt2Vector to){ - this.to = to; - } - - @Override - public NullableUInt2Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, char value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt2Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt2Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt2Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt2Vector to) { + this.to = to; + } + + @Override + public NullableUInt2Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt2Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java index 0527b10e1ea..1ec21490d84 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt4Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,260 +34,258 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt4Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 4; - private final FieldReader reader; - - public NullableUInt4Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), - allocator); - } - - public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt4ReaderImpl(NullableUInt4Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT4; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public int get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 4; + private final FieldReader reader; + + public NullableUInt4Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.UINT4.getType()), + allocator); + } + + public NullableUInt4Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt4ReaderImpl(NullableUInt4Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT4; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public int get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt4Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getInt(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Integer getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt4Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getInt(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Integer getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getInt(index * TYPE_WIDTH); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, int value) { - valueBuffer.setInt(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, int value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt4Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt4Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt4Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, int value) { + valueBuffer.setInt(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, int value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, int)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, int value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt4Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt4Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt4Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, int)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, int value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt4Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt4Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt4Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, int value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, int value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, int value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt4Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt4Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt4Vector to){ - this.to = to; - } - - @Override - public NullableUInt4Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, int value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt4Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt4Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt4Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt4Vector to) { + this.to = to; + } + + @Override + public NullableUInt4Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt4Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java index 4f28003e9f1..408a810cfa0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableUInt8Vector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,260 +34,258 @@ * maintained to track which elements in the vector are null. */ public class NullableUInt8Vector extends BaseNullableFixedWidthVector { - private static final byte TYPE_WIDTH = 8; - private final FieldReader reader; - - public NullableUInt8Vector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), - allocator); - } - - public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType, TYPE_WIDTH); - reader = new UInt8ReaderImpl(NullableUInt8Vector.this); - } - - @Override - public FieldReader getReader(){ - return reader; - } - - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.UINT8; - } - - - /****************************************************************** - * * - * vector value retrieval methods * - * * - ******************************************************************/ - - - /** - * Get the element at the given index from the vector. - * - * @param index position of element - * @return element at given index - */ - public long get(int index) throws IllegalStateException { - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + + public NullableUInt8Vector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.UINT8.getType()), + allocator); + } + + public NullableUInt8Vector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new UInt8ReaderImpl(NullableUInt8Vector.this); + } + + @Override + public FieldReader getReader() { + return reader; + } + + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.UINT8; + } + + + /****************************************************************** + * * + * vector value retrieval methods * + * * + ******************************************************************/ + + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public long get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + return valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableUInt8Holder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = valueBuffer.getLong(index * TYPE_WIDTH); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Long getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { return valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Get the element at the given index from the vector and - * sets the state in holder. If element at given index - * is null, holder.isSet will be zero. - * - * @param index position of element - */ - public void get(int index, NullableUInt8Holder holder){ - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.value = valueBuffer.getLong(index * TYPE_WIDTH); - } - - /** - * Same as {@link #get(int)}. - * - * @param index position of element - * @return element at given index - */ - public Long getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return valueBuffer.getLong(index * TYPE_WIDTH); - } - } - - public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { - if (from.isSet(fromIndex) != 0) { - set(thisIndex, from.get(fromIndex)); - } else { - BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); - } - } - - public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { - handleSafe(thisIndex); - copyFrom(fromIndex, thisIndex, from); - } - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - private void setValue(int index, long value) { - valueBuffer.setLong(index * TYPE_WIDTH, value); - } - - /** - * Set the element at the given index to the given value. - * - * @param index position of element - * @param value value of element - */ - public void set(int index, long value) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, value); - } - - /** - * Set the element at the given index to the value set in data holder. - * If the value in holder is not indicated as set, element in the - * at the given index will be null. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { - if(holder.isSet < 0) { - throw new IllegalArgumentException(); - } - else if(holder.isSet > 0) { - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - setValue(index, holder.value); - } - else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - /** - * Set the element at the given index to the value set in data holder. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void set(int index, UInt8Holder holder){ + } + } + + public void copyFrom(int fromIndex, int thisIndex, NullableUInt8Vector from) { + if (from.isSet(fromIndex) != 0) { + set(thisIndex, from.get(fromIndex)); + } else { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, 0); + } + } + + public void copyFromSafe(int fromIndex, int thisIndex, NullableUInt8Vector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + private void setValue(int index, long value) { + valueBuffer.setLong(index * TYPE_WIDTH, value); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, long value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); - } - - /** - * Same as {@link #set(int, long)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param value value of element - */ - public void setSafe(int index, long value) { - handleSafe(index); - set(index, value); - } - - /** - * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder nullable data holder for value of element - */ - public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { - handleSafe(index); - set(index, holder); - } - - /** - * Same as {@link #set(int, UInt8Holder)} except that it handles the - * case when index is greater than or equal to existing - * value capacity {@link #getValueCapacity()}. - * - * @param index position of element - * @param holder data holder for value of element - */ - public void setSafe(int index, UInt8Holder holder){ - handleSafe(index); - set(index, holder); - } - - /** - * Set the element at the given index to null. - * - * @param index position of element - */ - public void setNull(int index){ - handleSafe(index); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, UInt8Holder holder) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + setValue(index, holder.value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableUInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableUInt8Holder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, UInt8Holder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, UInt8Holder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); /* not really needed to set the bit to 0 as long as * the buffer always starts from 0. */ + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - - public void set(int index, int isSet, long value) { - if (isSet > 0) { - set(index, value); - } else { - BitVectorHelper.setValidityBit(validityBuffer, index, 0); - } - } - - public void setSafe(int index, int isSet, long value) { - handleSafe(index); - set(index, isSet, value); - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableUInt8Vector)to); - } - - private class TransferImpl implements TransferPair { - NullableUInt8Vector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableUInt8Vector to){ - this.to = to; - } - - @Override - public NullableUInt8Vector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); - } - } + } + } + + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableUInt8Vector) to); + } + + private class TransferImpl implements TransferPair { + NullableUInt8Vector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableUInt8Vector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableUInt8Vector to) { + this.to = to; + } + + @Override + public NullableUInt8Vector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableUInt8Vector.this); + } + } } \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java index 7a81733833e..1dfe8912c52 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarBinaryVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,294 +36,293 @@ * to track which elements in the vector are null. */ public class NullableVarBinaryVector extends BaseNullableVariableWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableVarBinaryVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); - } - - /** - * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType); - reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.VARBINARY; - } - - - /****************************************************************** - * * - * vector value getter methods * - * * - ******************************************************************/ - - - /** - * Get the variable length element at specified index as byte array. - * - * @param index position of element to get - * @return array of bytes for non-null element, null otherwise - */ - public byte[] get(int index) { - assert index >= 0; - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - final byte[] result = new byte[dataLength]; - valueBuffer.getBytes(startOffset, result, 0, dataLength); - return result; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return byte array for non-null element, null otherwise - */ - public byte[] getObject(int index) { - byte[] b; - try { - b = get(index); - } catch (IllegalStateException e) { - return null; - } - return b; - } - - /** - * Get the variable length element at specified index and sets the state - * in provided holder. - * - * @param index position of element to get - * @param holder data holder to be populated by this function - */ - public void get(int index, NullableVarBinaryHolder holder){ - assert index >= 0; - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.start = getstartOffset(index); - holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); - holder.buffer = valueBuffer; - } - - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { - final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); - final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); - final int length = end - start; - fillHoles(thisIndex); - BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); - final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); - from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); - offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); - lastSet = thisIndex; - } - - /** - * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { - final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); - final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); - final int length = end - start; - handleSafe(thisIndex, length); - fillHoles(thisIndex); - BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); - final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); - from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); - offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); - lastSet = thisIndex; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, VarBinaryHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, VarBinaryHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, VarBinaryHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, NullableVarBinaryHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, NullableVarBinaryHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarBinaryVector)to); - } - - private class TransferImpl implements TransferPair { - NullableVarBinaryVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableVarBinaryVector to){ - this.to = to; - } - - @Override - public NullableVarBinaryVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); - } - } + private final FieldReader reader; + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(Types.MinorType.VARBINARY.getType()), allocator); + } + + /** + * Instantiate a NullableVarBinaryVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarBinaryVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarBinaryReaderImpl(NullableVarBinaryVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARBINARY; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return byte array for non-null element, null otherwise + */ + public byte[] getObject(int index) { + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + return b; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + holder.buffer = valueBuffer; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarBinaryVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarBinaryVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarBinaryHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarBinaryHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarBinaryVector) to); + } + + private class TransferImpl implements TransferPair { + NullableVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarBinaryVector to) { + this.to = to; + } + + @Override + public NullableVarBinaryVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarBinaryVector.this); + } + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java index c1ebf4b7365..31e409cc626 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVarCharVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,296 +37,295 @@ * to track which elements in the vector are null. */ public class NullableVarCharVector extends BaseNullableVariableWidthVector { - private final FieldReader reader; - - /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param allocator allocator for memory management. - */ - public NullableVarCharVector(String name, BufferAllocator allocator) { - this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); - } - - /** - * Instantiate a NullableVarCharVector. This doesn't allocate any memory for - * the data in vector. - * @param name name of the vector - * @param fieldType type of Field materialized by this vector - * @param allocator allocator for memory management. - */ - public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { - super(name, allocator, fieldType); - reader = new VarCharReaderImpl(NullableVarCharVector.this); - } - - /** - * Get a reader that supports reading values from this vector - * @return Field Reader for this vector - */ - @Override - public FieldReader getReader(){ - return reader; - } - - /** - * Get minor type for this vector. The vector holds values belonging - * to a particular type. - * @return {@link org.apache.arrow.vector.types.Types.MinorType} - */ - @Override - public Types.MinorType getMinorType() { - return Types.MinorType.VARCHAR; - } - - - /****************************************************************** - * * - * vector value getter methods * - * * - ******************************************************************/ - - - /** - * Get the variable length element at specified index as byte array. - * - * @param index position of element to get - * @return array of bytes for non-null element, null otherwise - */ - public byte[] get(int index) { - assert index >= 0; - if(isSet(index) == 0) { - throw new IllegalStateException("Value at index is null"); - } - final int startOffset = getstartOffset(index); - final int dataLength = - offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; - final byte[] result = new byte[dataLength]; - valueBuffer.getBytes(startOffset, result, 0, dataLength); - return result; - } - - /** - * Get the variable length element at specified index as Text. - * - * @param index position of element to get - * @return Text object for non-null element, null otherwise - */ - public Text getObject(int index) { - Text result = new Text(); - byte[] b; - try { - b = get(index); - } catch (IllegalStateException e) { - return null; - } - result.set(b); - return result; - } - - /** - * Get the variable length element at specified index and sets the state - * in provided holder. - * - * @param index position of element to get - * @param holder data holder to be populated by this function - */ - public void get(int index, NullableVarCharHolder holder){ - assert index >= 0; - if(isSet(index) == 0) { - holder.isSet = 0; - return; - } - holder.isSet = 1; - holder.start = getstartOffset(index); - holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); - holder.buffer = valueBuffer; - } - - - - /****************************************************************** - * * - * vector value setter methods * - * * - ******************************************************************/ - - - /** - * Copy a cell value from a particular index in source vector to a particular - * position in this vector - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { - final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); - final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); - final int length = end - start; - fillHoles(thisIndex); - BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); - final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); - from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); - offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); - lastSet = thisIndex; - } - - /** - * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that - * it handles the case when the capacity of the vector needs to be expanded - * before copy. - * @param fromIndex position to copy from in source vector - * @param thisIndex position to copy to in this vector - * @param from source vector - */ - public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { - final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); - final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); - final int length = end - start; - handleSafe(thisIndex, length); - fillHoles(thisIndex); - BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); - final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); - from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); - offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); - lastSet = thisIndex; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, VarCharHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, VarCharHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, VarCharHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Set the variable length element at the specified index to the data - * buffer supplied in the holder - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void set(int index, NullableVarCharHolder holder) { - assert index >= 0; - fillHoles(index); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int dataLength = holder.end - holder.start; - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - /** - * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the - * case where index and length of new element are beyond the existing - * capacity of the vector. - * - * @param index position of the element to set - * @param holder holder that carries data buffer. - */ - public void setSafe(int index, NullableVarCharHolder holder) { - assert index >= 0; - final int dataLength = holder.end - holder.start; - fillEmpties(index); - handleSafe(index, dataLength); - BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); - final int startOffset = getstartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); - valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); - lastSet = index; - } - - - /****************************************************************** - * * - * vector transfer * - * * - ******************************************************************/ - - /** - * Construct a TransferPair comprising of this and and a target vector of - * the same type. - * @param ref name of the target vector - * @param allocator allocator for the target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator){ - return new TransferImpl(ref, allocator); - } - - /** - * Construct a TransferPair with a desired target vector of the same type. - * @param to target vector - * @return {@link TransferPair} - */ - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((NullableVarCharVector)to); - } - - private class TransferImpl implements TransferPair { - NullableVarCharVector to; - - public TransferImpl(String ref, BufferAllocator allocator){ - to = new NullableVarCharVector(ref, field.getFieldType(), allocator); - } - - public TransferImpl(NullableVarCharVector to){ - this.to = to; - } - - @Override - public NullableVarCharVector getTo(){ - return to; - } - - @Override - public void transfer(){ - transferTo(to); - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - splitAndTransferTo(startIndex, length, to); - } - - @Override - public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); - } - } + private final FieldReader reader; + + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param allocator allocator for memory management. + */ + public NullableVarCharVector(String name, BufferAllocator allocator) { + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.VARCHAR.getType()), allocator); + } + + /** + * Instantiate a NullableVarCharVector. This doesn't allocate any memory for + * the data in vector. + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public NullableVarCharVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType); + reader = new VarCharReaderImpl(NullableVarCharVector.this); + } + + /** + * Get a reader that supports reading values from this vector + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * @return {@link org.apache.arrow.vector.types.Types.MinorType} + */ + @Override + public Types.MinorType getMinorType() { + return Types.MinorType.VARCHAR; + } + + + /****************************************************************** + * * + * vector value getter methods * + * * + ******************************************************************/ + + + /** + * Get the variable length element at specified index as byte array. + * + * @param index position of element to get + * @return array of bytes for non-null element, null otherwise + */ + public byte[] get(int index) { + assert index >= 0; + if (isSet(index) == 0) { + throw new IllegalStateException("Value at index is null"); + } + final int startOffset = getstartOffset(index); + final int dataLength = + offsetBuffer.getInt((index + 1) * OFFSET_WIDTH) - startOffset; + final byte[] result = new byte[dataLength]; + valueBuffer.getBytes(startOffset, result, 0, dataLength); + return result; + } + + /** + * Get the variable length element at specified index as Text. + * + * @param index position of element to get + * @return Text object for non-null element, null otherwise + */ + public Text getObject(int index) { + Text result = new Text(); + byte[] b; + try { + b = get(index); + } catch (IllegalStateException e) { + return null; + } + result.set(b); + return result; + } + + /** + * Get the variable length element at specified index and sets the state + * in provided holder. + * + * @param index position of element to get + * @param holder data holder to be populated by this function + */ + public void get(int index, NullableVarCharHolder holder) { + assert index >= 0; + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.start = getstartOffset(index); + holder.end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + holder.buffer = valueBuffer; + } + + + /****************************************************************** + * * + * vector value setter methods * + * * + ******************************************************************/ + + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, NullableVarCharVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Same as {@link #copyFrom(int, int, NullableVarCharVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, NullableVarCharVector from) { + final int start = from.offsetBuffer.getInt(fromIndex * OFFSET_WIDTH); + final int end = from.offsetBuffer.getInt((fromIndex + 1) * OFFSET_WIDTH); + final int length = end - start; + handleSafe(thisIndex, length); + fillHoles(thisIndex); + BitVectorHelper.setValidityBit(this.validityBuffer, thisIndex, from.isSet(fromIndex)); + final int copyStart = offsetBuffer.getInt(thisIndex * OFFSET_WIDTH); + from.valueBuffer.getBytes(start, this.valueBuffer, copyStart, length); + offsetBuffer.setInt((thisIndex + 1) * OFFSET_WIDTH, copyStart + length); + lastSet = thisIndex; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, VarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, VarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, VarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Set the variable length element at the specified index to the data + * buffer supplied in the holder + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void set(int index, NullableVarCharHolder holder) { + assert index >= 0; + fillHoles(index); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int dataLength = holder.end - holder.start; + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + /** + * Same as {@link #set(int, NullableVarCharHolder)} except that it handles the + * case where index and length of new element are beyond the existing + * capacity of the vector. + * + * @param index position of the element to set + * @param holder holder that carries data buffer. + */ + public void setSafe(int index, NullableVarCharHolder holder) { + assert index >= 0; + final int dataLength = holder.end - holder.start; + fillEmpties(index); + handleSafe(index, dataLength); + BitVectorHelper.setValidityBit(validityBuffer, index, holder.isSet); + final int startOffset = getstartOffset(index); + offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); + lastSet = index; + } + + + /****************************************************************** + * * + * vector transfer * + * * + ******************************************************************/ + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((NullableVarCharVector) to); + } + + private class TransferImpl implements TransferPair { + NullableVarCharVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new NullableVarCharVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(NullableVarCharVector to) { + this.to = to; + } + + @Override + public NullableVarCharVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, NullableVarCharVector.this); + } + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index dc74afaa2e4..0d3acf1646f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 1489542b0fd..4648d078949 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,8 +74,7 @@ public boolean allocateNewSafe() { e.printStackTrace(); clear(); return false; - } - finally { + } finally { if (!dataAlloc) { clear(); } @@ -84,7 +83,7 @@ public boolean allocateNewSafe() { } protected void allocateOffsetBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; offsetBuffer = allocator.buffer(curSize); offsetBuffer.readerIndex(0); offsetAllocationSizeInBytes = curSize; @@ -101,8 +100,8 @@ protected void reallocOffsetBuffer() { final int currentBufferCapacity = offsetBuffer.capacity(); long baseSize = offsetAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -112,13 +111,13 @@ protected void reallocOffsetBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setBytes(0, offsetBuffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; newBuf.setZero(halfNewCapacity, halfNewCapacity); offsetBuffer.release(1); offsetBuffer = newBuf; - offsetAllocationSizeInBytes = (int)newAllocationSize; + offsetAllocationSizeInBytes = (int) newAllocationSize; } @Override @@ -148,7 +147,7 @@ public int getValueCapacity() { } protected int getOffsetBufferValueCapacity() { - return (int)((offsetBuffer.capacity() * 1.0)/OFFSET_WIDTH); + return (int) ((offsetBuffer.capacity() * 1.0) / OFFSET_WIDTH); } @Override @@ -188,7 +187,7 @@ public ArrowBuf[] getBuffers(boolean clear) { buffers = new ArrowBuf[0]; } else { buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false), - ArrowBuf.class); + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { @@ -213,15 +212,15 @@ public AddOrGetResult addOrGetVector(FieldType fieldT // returned vector must have the same field created = true; if (callBack != null && - // not a schema change if changing from ZeroVector to ZeroVector - (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { + // not a schema change if changing from ZeroVector to ZeroVector + (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { callBack.doWork(); } } if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); + fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); throw new SchemaChangeRuntimeException(msg); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 4c3b4b2a86c..6713b1c7871 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector.complex; @@ -125,7 +124,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers ArrowBuf bitBuffer = ownBuffers.get(0); validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); valueCount = fieldNode.getLength(); validityAllocationSizeInBytes = validityBuffer.capacity(); @@ -200,7 +199,7 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; @@ -217,8 +216,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -228,12 +227,12 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setZero(0, newBuf.capacity()); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } public FieldVector getDataVector() { @@ -292,7 +291,7 @@ public ArrowBuf[] getBuffers(boolean clear) { buffers = new ArrowBuf[0]; } else { buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false), - ArrowBuf.class); + ArrowBuf.class); } if (clear) { for (ArrowBuf buffer : buffers) { @@ -322,7 +321,7 @@ public AddOrGetResult addOrGetVector(FieldType type) // returned vector must have the same field if (!Objects.equals(vector.getField().getType(), type.getType())) { final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - type.getType(), vector.getField().getType()); + type.getType(), vector.getField().getType()); throw new SchemaChangeRuntimeException(msg); } @@ -411,7 +410,7 @@ public int getValueCount() { } private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); + return (int) (validityBuffer.capacity() * 8L); } public void setNull(int index) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java deleted file mode 100644 index 435d3d8ccf2..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyBaseRepeatedValueVector.java +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import java.util.Collections; -import java.util.Iterator; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.AddOrGetResult; -import org.apache.arrow.vector.BaseValueVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.UInt4Vector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - -public abstract class LegacyBaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector { - - public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; - public final static String DATA_VECTOR_NAME = "$data$"; - - protected final UInt4Vector offsets; - protected FieldVector vector; - protected final CallBack callBack; - - protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, DEFAULT_DATA_VECTOR, callBack); - } - - protected LegacyBaseRepeatedValueVector(String name, BufferAllocator allocator, FieldVector vector, CallBack callBack) { - super(name, allocator); - this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator); - this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); - this.callBack = callBack; - } - - @Override - public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - if (!offsets.allocateNewSafe()) { - return false; - } - success = vector.allocateNewSafe(); - } finally { - if (!success) { - clear(); - } - } - offsets.zeroVector(); - return success; - } - - @Override - public void reAlloc() { - offsets.reAlloc(); - vector.reAlloc(); - } - - @Override - public UInt4Vector getOffsetVector() { - return offsets; - } - - @Override - public FieldVector getDataVector() { - return vector; - } - - @Override - public void setInitialCapacity(int numRecords) { - offsets.setInitialCapacity(numRecords + 1); - vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); - } - - @Override - public int getValueCapacity() { - final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0); - if (vector == DEFAULT_DATA_VECTOR) { - return offsetValueCapacity; - } - return Math.min(vector.getValueCapacity(), offsetValueCapacity); - } - - @Override - public int getBufferSize() { - if (getAccessor().getValueCount() == 0) { - return 0; - } - return offsets.getBufferSize() + vector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(int valueCount) { - if (valueCount == 0) { - return 0; - } - - return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount); - } - - @Override - public Iterator iterator() { - return Collections.singleton(getDataVector()).iterator(); - } - - @Override - public void clear() { - offsets.clear(); - vector.clear(); - super.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class); - if (clear) { - for (ArrowBuf buffer : buffers) { - buffer.retain(); - } - clear(); - } - return buffers; - } - - /** - * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 - */ - public int size() { - return vector == DEFAULT_DATA_VECTOR ? 0 : 1; - } - - public AddOrGetResult addOrGetVector(FieldType fieldType) { - boolean created = false; - if (vector instanceof ZeroVector) { - vector = fieldType.createNewSingleVector(DATA_VECTOR_NAME, allocator, callBack); - // returned vector must have the same field - created = true; - if (callBack != null && - // not a schema change if changing from ZeroVector to ZeroVector - (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { - callBack.doWork(); - } - } - - if (vector.getField().getType().getTypeID() != fieldType.getType().getTypeID()) { - final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", - fieldType.getType().getTypeID(), vector.getField().getType().getTypeID()); - throw new SchemaChangeRuntimeException(msg); - } - - return new AddOrGetResult<>((T) vector, created); - } - - protected void replaceDataVector(FieldVector v) { - vector.clear(); - vector = v; - } - - public abstract class LegacyBaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { - - @Override - public int getValueCount() { - return Math.max(offsets.getAccessor().getValueCount() - 1, 0); - } - - @Override - public int getInnerValueCount() { - return vector.getAccessor().getValueCount(); - } - - @Override - public int getInnerValueCountAt(int index) { - return offsets.getAccessor().get(index + 1) - offsets.getAccessor().get(index); - } - - @Override - public boolean isNull(int index) { - return false; - } - - @Override - public boolean isEmpty(int index) { - return false; - } - } - - public abstract class LegacyBaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator { - - @Override - public int startNewValue(int index) { - while (offsets.getValueCapacity() <= index) { - offsets.reAlloc(); - } - int offset = offsets.getAccessor().get(index); - offsets.getMutator().setSafe(index + 1, offset); - setValueCount(index + 1); - return offset; - } - - @Override - public void setValueCount(int valueCount) { - // TODO: populate offset end points - offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount + 1); - final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount); - vector.getMutator().setValueCount(childValueCount); - } - } - -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java deleted file mode 100644 index 0843907d9df..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyFixedSizeListVector.java +++ /dev/null @@ -1,298 +0,0 @@ -/******************************************************************************* - - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.arrow.vector.complex; - -import static java.util.Collections.singletonList; -import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.SchemaChangeRuntimeException; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyFixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { - - public static LegacyFixedSizeListVector empty(String name, int size, BufferAllocator allocator) { - FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); - return new LegacyFixedSizeListVector(name, allocator, fieldType, null); - } - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); - /* delegate */ - private final FixedSizeListVector fixedSizeListVector; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyFixedSizeListVector(String name, - BufferAllocator allocator, - int listSize, - DictionaryEncoding dictionary, - CallBack schemaChangeCallback) { - super(name, allocator); - fixedSizeListVector = new FixedSizeListVector(name, allocator, listSize, dictionary, schemaChangeCallback); - } - - public LegacyFixedSizeListVector(String name, - BufferAllocator allocator, - FieldType fieldType, - CallBack schemaChangeCallback) { - super(name, allocator); - fixedSizeListVector = new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); - } - - @Override - public Field getField() { - return fixedSizeListVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.FIXED_SIZE_LIST; - } - - public int getListSize() { - return fixedSizeListVector.getListSize(); - } - - @Override - public void initializeChildrenFromFields(List children) { - fixedSizeListVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return fixedSizeListVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - fixedSizeListVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return fixedSizeListVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return fixedSizeListVector.getFieldInnerVectors(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public UnionFixedSizeListReader getReader() { - return fixedSizeListVector.getReader(); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - fixedSizeListVector.allocateNew(); - } - - @Override - public boolean allocateNewSafe() { - return fixedSizeListVector.allocateNewSafe(); - } - - @Override - public void reAlloc() { - fixedSizeListVector.reAlloc(); - } - - public FieldVector getDataVector() { - return fixedSizeListVector.getDataVector(); - } - - @Override - public void setInitialCapacity(int numRecords) { - fixedSizeListVector.setInitialCapacity(numRecords); - } - - @Override - public int getValueCapacity() { - return fixedSizeListVector.getValueCapacity(); - } - - @Override - public int getBufferSize() { - return fixedSizeListVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(int valueCount) { - return fixedSizeListVector.getBufferSizeFor(valueCount); - } - - @Override - public Iterator iterator() { - return fixedSizeListVector.iterator(); - } - - @Override - public void clear() { - fixedSizeListVector.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return fixedSizeListVector.getBuffers(clear); - } - - /** - * @return 1 if inner vector is explicitly set via #addOrGetVector else 0 - */ - public int size() { - return fixedSizeListVector.size(); - } - - @Override - @SuppressWarnings("unchecked") - public AddOrGetResult addOrGetVector(FieldType type) { - return fixedSizeListVector.addOrGetVector(type); - } - - public void copyFromSafe(int inIndex, int outIndex, FixedSizeListVector from) { - fixedSizeListVector.copyFromSafe(inIndex, outIndex, from); - } - - public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { - fixedSizeListVector.copyFrom(fromIndex, thisIndex, from); - } - - @Override - public UnionVector promoteToUnion() { - return fixedSizeListVector.promoteToUnion(); - } - - @Override - public long getValidityBufferAddress() { - return fixedSizeListVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return fixedSizeListVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return fixedSizeListVector.getObject(index); - } - - @Override - public boolean isNull(int index) { - return fixedSizeListVector.isNull(index); - } - - @Override - public int getNullCount() { - return fixedSizeListVector.getNullCount(); - } - - @Override - public int getValueCount() { - return fixedSizeListVector.getValueCount(); - } - } - - public class Mutator extends BaseValueVector.BaseMutator { - - public void setNull(int index) { - fixedSizeListVector.setNull(index); - } - - public void setNotNull(int index) { - fixedSizeListVector.setNotNull(index); - } - - @Override - public void setValueCount(int valueCount) { - fixedSizeListVector.setValueCount(valueCount); - } - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return fixedSizeListVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return fixedSizeListVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return fixedSizeListVector.makeTransferPair(((LegacyFixedSizeListVector)target).fixedSizeListVector); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java deleted file mode 100644 index 732699ada7a..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyListVector.java +++ /dev/null @@ -1,288 +0,0 @@ -/******************************************************************************* - - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.Collections.list; -import static java.util.Collections.singletonList; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.ComplexCopier; -import org.apache.arrow.vector.complex.impl.UnionListReader; -import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.complex.writer.FieldWriter; -import org.apache.arrow.vector.schema.ArrowFieldNode; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Null; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringArrayList; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyListVector extends LegacyBaseRepeatedValueVector implements FieldVector, PromotableVector { - - public static LegacyListVector empty(String name, BufferAllocator allocator) { - return new LegacyListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); - } - - private Mutator mutator = new Mutator(); - private Accessor accessor = new Accessor(); - - private final ListVector listVector; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyListVector(String name, BufferAllocator allocator, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, callBack); - } - - // deprecated, use FieldType or static constructor instead - @Deprecated - public LegacyListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, dictionary, callBack); - } - - public LegacyListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - listVector = new ListVector(name, allocator, fieldType, callBack); - } - - @Override - public void initializeChildrenFromFields(List children) { - listVector.initializeChildrenFromFields(children); - } - - @Override - public List getChildrenFromFields() { - return listVector.getChildrenFromFields(); - } - - @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - listVector.loadFieldBuffers(fieldNode, ownBuffers); - } - - @Override - public List getFieldBuffers() { - return listVector.getFieldBuffers(); - } - - @Override - public List getFieldInnerVectors() { - return listVector.getFieldInnerVectors(); - } - - public UnionListWriter getWriter() { - return listVector.getWriter(); - } - - @Override - public void allocateNew() throws OutOfMemoryException { - listVector.allocateNew(); - } - - @Override - public void reAlloc() { - listVector.reAlloc(); - } - - public void copyFromSafe(int inIndex, int outIndex, ListVector from) { - listVector.copyFrom(inIndex, outIndex, from); - } - - public void copyFrom(int inIndex, int outIndex, ListVector from) { - listVector.copyFrom(inIndex, outIndex, from); - } - - @Override - public FieldVector getDataVector() { - return listVector.getDataVector(); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return listVector.getTransferPair(ref, allocator); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return listVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector target) { - return listVector.makeTransferPair(((LegacyListVector)target).listVector); - } - - @Override - public long getValidityBufferAddress() { - return listVector.getValidityBufferAddress(); - } - - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); - } - - @Override - public long getOffsetBufferAddress() { - return listVector.getOffsetBufferAddress(); - } - - @Override - public ArrowBuf getValidityBuffer() { - return listVector.getValidityBuffer(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - return listVector.getOffsetBuffer(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - @Override - public UnionListReader getReader() { - return listVector.getReader(); - } - - @Override - public boolean allocateNewSafe() { - return listVector.allocateNewSafe(); - } - - public AddOrGetResult addOrGetVector(FieldType fieldType) { - return listVector.addOrGetVector(fieldType); - } - - @Override - public int getBufferSize() { - return listVector.getBufferSize(); - } - - @Override - public Field getField() { - return listVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.LIST; - } - - @Override - public void clear() { - listVector.clear(); - } - - @Override - public ArrowBuf[] getBuffers(boolean clear) { - return listVector.getBuffers(clear); - } - - @Override - public UnionVector promoteToUnion() { - return listVector.promoteToUnion(); - } - - private int lastSet = 0; - - public class Accessor extends LegacyBaseRepeatedAccessor { - - @Override - public Object getObject(int index) { - return listVector.getObject(index); - } - - @Override - public boolean isNull(int index) { - return listVector.isNull(index); - } - - @Override - public int getNullCount() { - return listVector.getNullCount(); - } - } - - public class Mutator extends LegacyBaseRepeatedMutator { - public void setNotNull(int index) { - listVector.setNotNull(index); - } - - @Override - public int startNewValue(int index) { - return listVector.startNewValue(index); - } - - /** - * End the current value - * - * @param index index of the value to end - * @param size number of elements in the list that was written - */ - public void endValue(int index, int size) { - listVector.endValue(index, size); - } - - @Override - public void setValueCount(int valueCount) { - listVector.setValueCount(valueCount); - } - - public void setLastSet(int value) { - listVector.setLastSet(value); - } - - public int getLastSet() { - return listVector.getLastSet(); - } - } - -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java deleted file mode 100644 index 77556ab774f..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LegacyMapVector.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; - -import io.netty.buffer.ArrowBuf; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringHashMap; -import org.apache.arrow.vector.util.TransferPair; - -public class LegacyMapVector extends AbstractMapVector { - - /* delegate */ - private final MapVector mapVector; - - public static LegacyMapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); - return new LegacyMapVector(name, allocator, fieldType, null); - } - - private final Accessor accessor = new Accessor(); - private final Mutator mutator = new Mutator(); - - @Deprecated - public LegacyMapVector(String name, BufferAllocator allocator, CallBack callBack) { - super(name, allocator, callBack); - mapVector = new MapVector(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - } - - public LegacyMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - mapVector = new MapVector(name, allocator, fieldType, callBack); - } - - @Override - public FieldReader getReader() { - return mapVector.getReader(); - } - - public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { - mapVector.copyFromSafe(fromIndex, thisIndex, from); - } - - @Override - protected boolean supportsDirectRead() { - return true; - } - - public Iterator fieldNameIterator() { - return mapVector.fieldNameIterator(); - } - - @Override - public void setInitialCapacity(int numRecords) { - mapVector.setInitialCapacity(numRecords); - } - - @Override - public int getBufferSize() { - return mapVector.getBufferSize(); - } - - @Override - public int getBufferSizeFor(final int valueCount) { - return (int) mapVector.getBufferSizeFor(valueCount); - } - - @Override - public ArrowBuf getValidityBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(name, allocator, null); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return mapVector.getTransferPair(ref, allocator, callBack); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return mapVector.makeTransferPair(((LegacyMapVector)to).mapVector); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return mapVector.getTransferPair(ref, allocator); - } - - @Override - public int getValueCapacity() { - return mapVector.getValueCapacity(); - } - - @Override - public Accessor getAccessor() { - return accessor; - } - - @Override - public Mutator getMutator() { - return mutator; - } - - public class Accessor extends BaseValueVector.BaseAccessor { - - @Override - public Object getObject(int index) { - return mapVector.getObject(index); - } - - public void get(int index, ComplexHolder holder) { - mapVector.get(index, holder); - } - - @Override - public int getValueCount() { - return mapVector.getValueCount(); - } - } - - public ValueVector getVectorById(int id) { - return mapVector.getVectorById(id); - } - - public class Mutator extends BaseValueVector.BaseMutator { - - @Override - public void setValueCount(int valueCount) { - mapVector.setValueCount(valueCount); - } - - @Override - public void reset() { } - - @Override - public void generateTestData(int values) { } - } - - @Override - public void clear() { - mapVector.clear(); - } - - @Override - public Field getField() { - return mapVector.getField(); - } - - @Override - public MinorType getMinorType() { - return MinorType.MAP; - } - - @Override - public void close() { - mapVector.close(); - } - - public void initializeChildrenFromFields(List children) { - mapVector.initializeChildrenFromFields(children); - } - - public List getChildrenFromFields() { - return mapVector.getChildren(); - } - - public boolean isNull(int index) { return false; } - - public int getNullCount() { return 0; } - - public int getValueCount() { return 0; } - - public void setValueCount(int valueCount) { } - - public Object getObject(int index) { return null; } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 1134ac88cd5..3cd8bf9ce47 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,15 +6,15 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.arrow.vector.complex; @@ -163,9 +162,9 @@ public UnionListWriter getWriter() { @Override public void allocateNew() throws OutOfMemoryException { - if (!allocateNewSafe()) { - throw new OutOfMemoryException("Failure while allocating memory"); - } + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } } public boolean allocateNewSafe() { @@ -187,7 +186,7 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; @@ -211,8 +210,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -222,13 +221,13 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); final int halfNewCapacity = newBuf.capacity() / 2; newBuf.setZero(halfNewCapacity, halfNewCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } public void copyFromSafe(int inIndex, int outIndex, ListVector from) { @@ -279,7 +278,9 @@ public long getOffsetBufferAddress() { } @Override - public ArrowBuf getValidityBuffer() { return validityBuffer; } + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } @Override public ArrowBuf getDataBuffer() { @@ -287,7 +288,9 @@ public ArrowBuf getDataBuffer() { } @Override - public ArrowBuf getOffsetBuffer() { return offsetBuffer; } + public ArrowBuf getOffsetBuffer() { + return offsetBuffer; + } private class TransferImpl implements TransferPair { @@ -357,8 +360,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect } target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); - } - else { + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. @@ -383,15 +385,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListVect * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { + } else { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); @@ -524,7 +525,7 @@ public int getValidityAndOffsetValueCapacity() { } private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); + return (int) (validityBuffer.capacity() * 8L); } public void setNotNull(int index) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 1c0e5df4de6..533efc9063c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -91,7 +91,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers ArrowBuf bitBuffer = ownBuffers.get(0); validityBuffer.release(); - validityBuffer = bitBuffer.retain(allocator); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); valueCount = fieldNode.getLength(); validityAllocationSizeInBytes = validityBuffer.capacity(); } @@ -197,8 +197,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable } target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); - } - else { + } else { /* Copy data * When the first bit starts from the middle of a byte (offset != 0), * copy data from src BitVector. @@ -223,15 +222,14 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable * (we are at the last byte), we copy the last piece as a byte * by shifting data from the current byte. */ - if((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } - else { + } else { byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1); @@ -241,7 +239,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable } private int getValidityBufferValueCapacity() { - return (int)(validityBuffer.capacity() * 8L); + return (int) (validityBuffer.capacity() * 8L); } @Override @@ -261,7 +259,7 @@ public ArrowBuf[] getBuffers(boolean clear) { ArrowBuf.class); } if (clear) { - for (ArrowBuf buffer: buffers) { + for (ArrowBuf buffer : buffers) { buffer.retain(); } clear(); @@ -289,7 +287,9 @@ private void clearValidityBuffer() { @Override public int getBufferSize() { - if (valueCount == 0) { return 0; } + if (valueCount == 0) { + return 0; + } return super.getBufferSize() + BitVectorHelper.getValidityBufferSize(valueCount); } @@ -300,7 +300,7 @@ public int getBufferSizeFor(final int valueCount) { return 0; } return super.getBufferSizeFor(valueCount) - + BitVectorHelper.getValidityBufferSize(valueCount); + + BitVectorHelper.getValidityBufferSize(valueCount); } @Override @@ -331,7 +331,7 @@ public boolean allocateNewSafe() { } private void allocateValidityBuffer(final long size) { - final int curSize = (int)size; + final int curSize = (int) size; validityBuffer = allocator.buffer(curSize); validityBuffer.readerIndex(0); validityAllocationSizeInBytes = curSize; @@ -349,8 +349,8 @@ private void reallocValidityBuffer() { final int currentBufferCapacity = validityBuffer.capacity(); long baseSize = validityAllocationSizeInBytes; - if (baseSize < (long)currentBufferCapacity) { - baseSize = (long)currentBufferCapacity; + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; } long newAllocationSize = baseSize * 2L; @@ -360,12 +360,12 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize); + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setZero(0, newBuf.capacity()); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); validityBuffer.release(1); validityBuffer = newBuf; - validityAllocationSizeInBytes = (int)newAllocationSize; + validityAllocationSizeInBytes = (int) newAllocationSize; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 61a591d81fd..dc325b4920e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -193,7 +193,7 @@ public VectorSchemaRoot read() throws IOException { private abstract class BufferReader { abstract protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException; - final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { + ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOException { readToken(START_ARRAY); ArrowBuf buf = read(allocator, count); readToken(END_ARRAY); @@ -202,8 +202,7 @@ final ArrowBuf readBuffer(BufferAllocator allocator, int count) throws IOExcepti } private class BufferHelper { - BufferReader BIT = new BufferReader() { - + BufferReader BIT = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { final int bufferSize = BitVectorHelper.getValidityBufferSize(count); @@ -217,6 +216,7 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0); } + buf.writerIndex(bufferSize); return buf; } }; @@ -224,13 +224,15 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT1 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableTinyIntVector.TYPE_WIDTH); + final int size = count * NullableTinyIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeByte(parser.getByteValue()); } + buf.writerIndex(size); return buf; } }; @@ -238,88 +240,96 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException BufferReader INT2 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableSmallIntVector.TYPE_WIDTH); + final int size = count * NullableSmallIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeShort(parser.getShortValue()); } + buf.writerIndex(size); return buf; } }; BufferReader INT4 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableIntVector.TYPE_WIDTH); + final int size = count * NullableIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeInt(parser.getIntValue()); } + buf.writerIndex(size); return buf; } }; BufferReader INT8 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableBigIntVector.TYPE_WIDTH); + final int size = count * NullableBigIntVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeLong(parser.getLongValue()); } + buf.writerIndex(size); return buf; } }; BufferReader FLOAT4 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableFloat4Vector.TYPE_WIDTH); + final int size = count * NullableFloat4Vector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeFloat(parser.getFloatValue()); } + buf.writerIndex(size); return buf; } }; BufferReader FLOAT8 = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableFloat8Vector.TYPE_WIDTH); + final int size = count * NullableFloat8Vector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); for (int i = 0; i < count; i++) { parser.nextToken(); buf.writeDouble(parser.getDoubleValue()); } + buf.writerIndex(size); return buf; } }; BufferReader DECIMAL = new BufferReader() { - @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { - ArrowBuf buf = allocator.buffer(count * NullableDecimalVector.TYPE_WIDTH); + final int size = count * NullableDecimalVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); + for (int i = 0; i < count; i++) { parser.nextToken(); BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i); } + buf.writerIndex(size); return buf; } }; @@ -369,10 +379,10 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException return buf; } }; - } - private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferType, Types.MinorType type, int count) throws IOException { + private ArrowBuf readIntoBuffer(BufferAllocator allocator, ArrowVectorType bufferType, + Types.MinorType type, int count) throws IOException { ArrowBuf buf; BufferHelper helper = new BufferHelper(); @@ -461,6 +471,7 @@ private ArrowBuf readBuffer(BufferAllocator allocator, ArrowVectorType bufferTyp } buf = reader.readBuffer(allocator, count); + assert buf != null; return buf; } @@ -506,12 +517,21 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json innerBufferValueCount = valueCount + 1; } - vectorBuffers[v] = readBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); + vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); + + if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || + vector.getMinorType() == Types.MinorType.VARBINARY)) { + final ArrowBuf offsetBuffer = vectorBuffers[v-1]; + final int lastOffset = offsetBuffer.getInt(valueCount * 4); + vectorBuffers[v].writerIndex(lastOffset); + } } - vector.loadFieldBuffers(new ArrowFieldNode(valueCount, 0), Arrays.asList(vectorBuffers)); + final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount); + final ArrowFieldNode fieldNode = new ArrowFieldNode(valueCount, nullCount); + vector.loadFieldBuffers(fieldNode, Arrays.asList(vectorBuffers)); - // read child vectors, if any + /* read child vectors (if any) */ List fields = field.getChildren(); if (!fields.isEmpty()) { List vectorChildren = vector.getChildrenFromFields(); From 44b9dc1e21d5afb4f370949e55fcc957ebcb3da3 Mon Sep 17 00:00:00 2001 From: siddharth Date: Wed, 15 Nov 2017 10:40:06 -0800 Subject: [PATCH 09/10] review comments --- .../arrow/vector/GenerateSampleData.java | 532 +++++++++--------- .../apache/arrow/vector/VectorUnloader.java | 3 - .../arrow/vector/complex/ListVector.java | 2 +- .../vector/file/json/JsonFileReader.java | 13 - 4 files changed, 267 insertions(+), 283 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java index 4be5d44d090..806beb5f8fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,308 +29,308 @@ */ public class GenerateSampleData { - public static void generateTestData(final ValueVector vector, final int valueCount) { - if (vector instanceof NullableIntVector) { - writeIntData((NullableIntVector)vector, valueCount); - } else if (vector instanceof NullableDecimalVector) { - writeDecimalData((NullableDecimalVector)vector, valueCount); - } else if (vector instanceof NullableBitVector) { - writeBooleanData((NullableBitVector)vector, valueCount); - } else if (vector instanceof NullableVarCharVector) { - writeVarCharData((NullableVarCharVector)vector, valueCount); - } else if (vector instanceof NullableVarBinaryVector) { - writeVarBinaryData((NullableVarBinaryVector)vector, valueCount); - } else if (vector instanceof NullableBigIntVector) { - writeBigIntData((NullableBigIntVector) vector, valueCount); - } else if (vector instanceof NullableFloat4Vector) { - writeFloatData((NullableFloat4Vector)vector, valueCount); - } else if (vector instanceof NullableFloat8Vector) { - writeDoubleData((NullableFloat8Vector) vector, valueCount); - } else if (vector instanceof NullableDateDayVector) { - writeDateDayData((NullableDateDayVector)vector, valueCount); - } else if (vector instanceof NullableDateMilliVector) { - writeDateMilliData((NullableDateMilliVector) vector, valueCount); - } else if (vector instanceof NullableIntervalDayVector) { - writeIntervalDayData((NullableIntervalDayVector)vector, valueCount); - } else if (vector instanceof NullableIntervalYearVector) { - writeIntervalYearData((NullableIntervalYearVector) vector, valueCount); - } else if (vector instanceof NullableSmallIntVector) { - writeSmallIntData((NullableSmallIntVector)vector, valueCount); - } else if (vector instanceof NullableTinyIntVector) { - writeTinyIntData((NullableTinyIntVector) vector, valueCount); - } else if (vector instanceof NullableTimeMicroVector) { - writeTimeMicroData((NullableTimeMicroVector)vector, valueCount); - } else if (vector instanceof NullableTimeMilliVector) { - writeTimeMilliData((NullableTimeMilliVector) vector, valueCount); - } else if (vector instanceof NullableTimeNanoVector) { - writeTimeNanoData((NullableTimeNanoVector)vector, valueCount); - } else if (vector instanceof NullableTimeSecVector) { - writeTimeSecData((NullableTimeSecVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampSecVector) { - writeTimeStampData((NullableTimeStampSecVector)vector, valueCount); - } else if (vector instanceof NullableTimeStampMicroVector) { - writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMilliVector) { - writeTimeStampData((NullableTimeStampMilliVector)vector, valueCount); - } else if (vector instanceof NullableTimeStampNanoVector) { - writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampSecTZVector) { - writeTimeStampData((NullableTimeStampSecTZVector)vector, valueCount); - } else if (vector instanceof NullableTimeStampMicroTZVector) { - writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount); - } else if (vector instanceof NullableTimeStampMilliTZVector) { - writeTimeStampData((NullableTimeStampMilliTZVector)vector, valueCount); - } else if (vector instanceof NullableTimeStampNanoTZVector) { - writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount); - } - } + public static void generateTestData(final ValueVector vector, final int valueCount) { + if (vector instanceof NullableIntVector) { + writeIntData((NullableIntVector) vector, valueCount); + } else if (vector instanceof NullableDecimalVector) { + writeDecimalData((NullableDecimalVector) vector, valueCount); + } else if (vector instanceof NullableBitVector) { + writeBooleanData((NullableBitVector) vector, valueCount); + } else if (vector instanceof NullableVarCharVector) { + writeVarCharData((NullableVarCharVector) vector, valueCount); + } else if (vector instanceof NullableVarBinaryVector) { + writeVarBinaryData((NullableVarBinaryVector) vector, valueCount); + } else if (vector instanceof NullableBigIntVector) { + writeBigIntData((NullableBigIntVector) vector, valueCount); + } else if (vector instanceof NullableFloat4Vector) { + writeFloatData((NullableFloat4Vector) vector, valueCount); + } else if (vector instanceof NullableFloat8Vector) { + writeDoubleData((NullableFloat8Vector) vector, valueCount); + } else if (vector instanceof NullableDateDayVector) { + writeDateDayData((NullableDateDayVector) vector, valueCount); + } else if (vector instanceof NullableDateMilliVector) { + writeDateMilliData((NullableDateMilliVector) vector, valueCount); + } else if (vector instanceof NullableIntervalDayVector) { + writeIntervalDayData((NullableIntervalDayVector) vector, valueCount); + } else if (vector instanceof NullableIntervalYearVector) { + writeIntervalYearData((NullableIntervalYearVector) vector, valueCount); + } else if (vector instanceof NullableSmallIntVector) { + writeSmallIntData((NullableSmallIntVector) vector, valueCount); + } else if (vector instanceof NullableTinyIntVector) { + writeTinyIntData((NullableTinyIntVector) vector, valueCount); + } else if (vector instanceof NullableTimeMicroVector) { + writeTimeMicroData((NullableTimeMicroVector) vector, valueCount); + } else if (vector instanceof NullableTimeMilliVector) { + writeTimeMilliData((NullableTimeMilliVector) vector, valueCount); + } else if (vector instanceof NullableTimeNanoVector) { + writeTimeNanoData((NullableTimeNanoVector) vector, valueCount); + } else if (vector instanceof NullableTimeSecVector) { + writeTimeSecData((NullableTimeSecVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecVector) { + writeTimeStampData((NullableTimeStampSecVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroVector) { + writeTimeStampData((NullableTimeStampMicroVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliVector) { + writeTimeStampData((NullableTimeStampMilliVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoVector) { + writeTimeStampData((NullableTimeStampNanoVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampSecTZVector) { + writeTimeStampData((NullableTimeStampSecTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMicroTZVector) { + writeTimeStampData((NullableTimeStampMicroTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampMilliTZVector) { + writeTimeStampData((NullableTimeStampMilliTZVector) vector, valueCount); + } else if (vector instanceof NullableTimeStampNanoTZVector) { + writeTimeStampData((NullableTimeStampNanoTZVector) vector, valueCount); + } + } - private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) { - final long even = 100000; - final long odd = 200000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTimeStampData(NullableTimeStampVector vector, int valueCount) { + final long even = 100000; + final long odd = 200000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeDecimalData(NullableDecimalVector vector, int valueCount) { - final BigDecimal even = new BigDecimal(0.0543278923); - final BigDecimal odd = new BigDecimal(2.0543278923); - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeDecimalData(NullableDecimalVector vector, int valueCount) { + final BigDecimal even = new BigDecimal(0.0543278923); + final BigDecimal odd = new BigDecimal(2.0543278923); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeIntData(NullableIntVector vector, int valueCount) { - final int even = 1000; - final int odd = 2000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeIntData(NullableIntVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeBooleanData(NullableBitVector vector, int valueCount) { - final int even = 0; - final int odd = 1; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeBooleanData(NullableBitVector vector, int valueCount) { + final int even = 0; + final int odd = 1; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) { - final int even = 1; - final int odd = 2; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeIntervalYearData(NullableIntervalYearVector vector, int valueCount) { + final int even = 1; + final int odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) { - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, 1, 50); - } else { - vector.setSafe(i, 2, 100); - } + private static void writeIntervalDayData(NullableIntervalDayVector vector, int valueCount) { + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, 1, 50); + } else { + vector.setSafe(i, 2, 100); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) { - final int even = 500; - final int odd = 900; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTimeSecData(NullableTimeSecVector vector, int valueCount) { + final int even = 500; + final int odd = 900; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) { - final int even = 1000; - final int odd = 2000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTimeMilliData(NullableTimeMilliVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) { - final long even = 1000000000; - final long odd = 2000000000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTimeMicroData(NullableTimeMicroVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); + } + vector.setValueCount(valueCount); - } + } - private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) { - final long even = 1000000000; - final long odd = 2000000000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTimeNanoData(NullableTimeNanoVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeDateDayData(NullableDateDayVector vector, int valueCount) { - final int even = 1000; - final int odd = 2000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeDateDayData(NullableDateDayVector vector, int valueCount) { + final int even = 1000; + final int odd = 2000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) { - final long even = 1000000000; - final long odd = 2000000000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeDateMilliData(NullableDateMilliVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) { - final short even = 10; - final short odd = 20; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeSmallIntData(NullableSmallIntVector vector, int valueCount) { + final short even = 10; + final short odd = 20; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) { - final byte even = 1; - final byte odd = 2; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeTinyIntData(NullableTinyIntVector vector, int valueCount) { + final byte even = 1; + final byte odd = 2; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeBigIntData(NullableBigIntVector vector, int valueCount) { - final long even = 1000000000; - final long odd = 2000000000; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeBigIntData(NullableBigIntVector vector, int valueCount) { + final long even = 1000000000; + final long odd = 2000000000; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeFloatData(NullableFloat4Vector vector, int valueCount) { - final float even = 20.3f; - final float odd = 40.2f; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeFloatData(NullableFloat4Vector vector, int valueCount) { + final float even = 20.3f; + final float odd = 40.2f; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) { - final double even = 20.2373; - final double odd = 40.2378; - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeDoubleData(NullableFloat8Vector vector, int valueCount) { + final double even = 20.2373; + final double odd = 40.2378; + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) { - Charset utf8Charset = Charset.forName("UTF-8"); - final byte[] even = "AAAAA1".getBytes(utf8Charset); - final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeVarBinaryData(NullableVarBinaryVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } - private static void writeVarCharData(NullableVarCharVector vector, int valueCount) { - Charset utf8Charset = Charset.forName("UTF-8"); - final byte[] even = "AAAAA1".getBytes(utf8Charset); - final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); - for (int i = 0; i < valueCount; i++) { - if (i%2 == 0) { - vector.setSafe(i, even); - } else { - vector.setSafe(i, odd); - } + private static void writeVarCharData(NullableVarCharVector vector, int valueCount) { + Charset utf8Charset = Charset.forName("UTF-8"); + final byte[] even = "AAAAA1".getBytes(utf8Charset); + final byte[] odd = "BBBBBBBBB2".getBytes(utf8Charset); + for (int i = 0; i < valueCount; i++) { + if (i % 2 == 0) { + vector.setSafe(i, even); + } else { + vector.setSafe(i, odd); } - vector.setValueCount(valueCount); - } + } + vector.setValueCount(valueCount); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 288afc6bbec..fd9677312ed 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -22,13 +22,10 @@ import java.util.List; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.vector.ValueVector.Accessor; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.schema.ArrowVectorType; -import javax.annotation.Nullable; - public class VectorUnloader { private final VectorSchemaRoot root; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 3cd8bf9ce47..d626c9bdbf5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -519,7 +519,7 @@ public int getValueCapacity() { return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); } - public int getValidityAndOffsetValueCapacity() { + private int getValidityAndOffsetValueCapacity() { final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); return Math.min(offsetValueCapacity, getValidityBufferValueCapacity()); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index dc325b4920e..8017b385d14 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -232,7 +232,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeByte(parser.getByteValue()); } - buf.writerIndex(size); return buf; } }; @@ -248,7 +247,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeShort(parser.getShortValue()); } - buf.writerIndex(size); return buf; } }; @@ -264,7 +262,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeInt(parser.getIntValue()); } - buf.writerIndex(size); return buf; } }; @@ -280,7 +277,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeLong(parser.getLongValue()); } - buf.writerIndex(size); return buf; } }; @@ -296,7 +292,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeFloat(parser.getFloatValue()); } - buf.writerIndex(size); return buf; } }; @@ -312,7 +307,6 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException buf.writeDouble(parser.getDoubleValue()); } - buf.writerIndex(size); return buf; } }; @@ -518,13 +512,6 @@ private void readFromJsonIntoVector(Field field, FieldVector vector) throws Json } vectorBuffers[v] = readIntoBuffer(allocator, vectorType, vector.getMinorType(), innerBufferValueCount); - - if (vectorType.equals(DATA) && (vector.getMinorType() == Types.MinorType.VARCHAR || - vector.getMinorType() == Types.MinorType.VARBINARY)) { - final ArrowBuf offsetBuffer = vectorBuffers[v-1]; - final int lastOffset = offsetBuffer.getInt(valueCount * 4); - vectorBuffers[v].writerIndex(lastOffset); - } } final int nullCount = BitVectorHelper.getNullCount(vectorBuffers[0], valueCount); From 77c3ba321b337221603712fccc5cdaad86040dd1 Mon Sep 17 00:00:00 2001 From: siddharth Date: Wed, 15 Nov 2017 11:27:28 -0800 Subject: [PATCH 10/10] indentation and javadocs --- .../vector/BaseNullableFixedWidthVector.java | 132 +++++++++--------- .../BaseNullableVariableWidthVector.java | 128 ++++++++--------- .../apache/arrow/vector/BitVectorHelper.java | 14 +- .../arrow/vector/complex/ListVector.java | 110 +++++++++++++++ .../vector/complex/NullableMapVector.java | 39 ++++++ 5 files changed, 287 insertions(+), 136 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java index a4b48de10a8..209758e4ece 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableFixedWidthVector.java @@ -66,31 +66,30 @@ public BaseNullableFixedWidthVector(final String name, final BufferAllocator all valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth; validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } else { - /* specialized handling for NullableBitVector */ + /* specialized handling for NullableBitVector */ valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); validityAllocationSizeInBytes = valueAllocationSizeInBytes; } } - /* TODO: - * - * see if getNullCount() can be made faster -- O(1) - */ + /* TODO: + * see if getNullCount() can be made faster -- O(1) + */ - /* TODO: - * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level - * base class BaseValueVector. - * - * Along with this, some class members (validityBuffer) can also be - * abstracted out to top level base class. - * - * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy (non-nullable) and those - * vectors have not yet been refactored/removed so moving things to - * the top class as of now is not a good idea. - */ + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. + */ @Override @@ -312,7 +311,7 @@ public void allocateNew(int valueCount) { long valueBufferSize = valueCount * typeWidth; long validityBufferSize = getValidityBufferSizeFromCount(valueCount); if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ + /* specialized handling for NullableBitVector */ valueBufferSize = validityBufferSize; } @@ -320,7 +319,7 @@ public void allocateNew(int valueCount) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } - /* we are doing a new allocation -- release the current buffers */ + /* we are doing a new allocation -- release the current buffers */ clear(); try { @@ -342,12 +341,12 @@ public void allocateNew(int valueCount) { * conditions. */ private void allocateBytes(final long valueBufferSize, final long validityBufferSize) { - /* allocate data buffer */ + /* allocate data buffer */ int curSize = (int) valueBufferSize; valueBuffer = allocator.buffer(curSize); valueBuffer.readerIndex(0); valueAllocationSizeInBytes = curSize; - /* allocate validity buffer */ + /* allocate validity buffer */ allocateValidityBuffer((int) validityBufferSize); zeroVector(); } @@ -545,6 +544,9 @@ public List getFieldBuffers() { return result; } + /** + * Set the reader and writer indexes for the inner buffers. + */ private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); valueBuffer.readerIndex(0); @@ -554,7 +556,7 @@ private void setReaderAndWriterIndex() { } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); if (typeWidth == 0) { - /* specialized handling for NullableBitVector */ + /* specialized handling for NullableBitVector */ valueBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); } else { valueBuffer.writerIndex(valueCount * typeWidth); @@ -647,19 +649,19 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, if (length > 0) { if (offset == 0) { - /* slice */ + /* slice */ if (target.validityBuffer != null) { target.validityBuffer.release(); } target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { @@ -671,15 +673,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.validityBuffer.setByte(i, (b1 + b2)); } - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); @@ -734,32 +736,32 @@ public void setValueCount(int valueCount) { while (valueCount > getValueCapacity()) { reAlloc(); } - /* - * We are trying to understand the pattern of memory allocation. - * If initially, the user did vector.allocateNew(), we would have - * allocated memory of default size (4096 * type width). - * Later on user invokes setValueCount(count). - * - * If the existing value capacity is twice as large as the - * valueCount, we know that we over-provisioned memory in the - * first place when default memory allocation was done because user - * really needs a much less value count in the vector. - * - * We record this by bumping up the allocationMonitor. If this pattern - * happens for certain number of times and allocationMonitor - * reaches the threshold (internal hardcoded) value, subsequent - * call to allocateNew() will take care of stepping down the - * default memory allocation size. - * - * Another case would be under-provisioning the initial memory and - * thus going through a lot of realloc(). Here the goal is to - * see if we can minimize the number of reallocations. Again the - * state is recorded in allocationMonitor by decrementing it - * (negative value). If a threshold is hit, realloc will try to - * allocate more memory in order to possibly avoid a future realloc. - * This case is also applicable to setSafe() methods which can trigger - * a realloc() and thus we record the state there as well. - */ + /* + * We are trying to understand the pattern of memory allocation. + * If initially, the user did vector.allocateNew(), we would have + * allocated memory of default size (4096 * type width). + * Later on user invokes setValueCount(count). + * + * If the existing value capacity is twice as large as the + * valueCount, we know that we over-provisioned memory in the + * first place when default memory allocation was done because user + * really needs a much less value count in the vector. + * + * We record this by bumping up the allocationMonitor. If this pattern + * happens for certain number of times and allocationMonitor + * reaches the threshold (internal hardcoded) value, subsequent + * call to allocateNew() will take care of stepping down the + * default memory allocation size. + * + * Another case would be under-provisioning the initial memory and + * thus going through a lot of realloc(). Here the goal is to + * see if we can minimize the number of reallocations. Again the + * state is recorded in allocationMonitor by decrementing it + * (negative value). If a threshold is hit, realloc will try to + * allocate more memory in order to possibly avoid a future realloc. + * This case is also applicable to setSafe() methods which can trigger + * a realloc() and thus we record the state there as well. + */ if (valueCount > 0) { if (currentValueCapacity >= (valueCount * 2)) { incrementAllocationMonitor(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java index f819ab1c4e7..edf4987de57 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseNullableVariableWidthVector.java @@ -72,25 +72,23 @@ public BaseNullableVariableWidthVector(final String name, final BufferAllocator cleared = false; } - /* TODO: - * - * see if getNullCount() can be made faster -- O(1) - */ - /* TODO: - * Once the entire hierarchy has been refactored, move common functions - * like getNullCount(), splitAndTransferValidityBuffer to top level - * base class BaseValueVector. - * - * Along with this, some class members (validityBuffer) can also be - * abstracted out to top level base class. - * - * Right now BaseValueVector is the top level base class for other - * vector types in ValueVector hierarchy (non-nullable) and those - * vectors have not yet been refactored/removed so moving things to - * the top class as of now is not a good idea. - */ + * see if getNullCount() can be made faster -- O(1) + */ + /* TODO: + * Once the entire hierarchy has been refactored, move common functions + * like getNullCount(), splitAndTransferValidityBuffer to top level + * base class BaseValueVector. + * + * Along with this, some class members (validityBuffer) can also be + * abstracted out to top level base class. + * + * Right now BaseValueVector is the top level base class for other + * vector types in ValueVector hierarchy (non-nullable) and those + * vectors have not yet been refactored/removed so moving things to + * the top class as of now is not a good idea. + */ @Override @Deprecated @@ -177,9 +175,9 @@ public void setInitialCapacity(int valueCount) { } valueAllocationSizeInBytes = (int) size; validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount); - /* to track the end offset of last data element in vector, we need - * an additional slot in offset buffer. - */ + /* to track the end offset of last data element in vector, we need + * an additional slot in offset buffer. + */ offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH; } @@ -317,6 +315,9 @@ public List getFieldBuffers() { return result; } + /** + * Set the reader and writer indexes for the inner buffers. + */ private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); @@ -362,7 +363,7 @@ public boolean allocateNewSafe() { throw new OversizedAllocationException("Requested amount of memory exceeds limit"); } - /* we are doing a new allocation -- release the current buffers */ + /* we are doing a new allocation -- release the current buffers */ clear(); try { @@ -395,7 +396,7 @@ public void allocateNew(int totalBytes, int valueCount) { throw new OversizedAllocationException("Requested amount of memory exceeds limit"); } - /* we are doing a new allocation -- release the current buffers */ + /* we are doing a new allocation -- release the current buffers */ clear(); try { @@ -409,7 +410,7 @@ public void allocateNew(int totalBytes, int valueCount) { /* allocate the inner buffers */ private void allocateBytes(final long valueBufferSize, final long validityBufferSize, final long offsetBufferSize) { - /* allocate data buffer */ + /* allocate data buffer */ int curSize = (int) valueBufferSize; valueBuffer = allocator.buffer(curSize); valueBuffer.readerIndex(0); @@ -728,12 +729,12 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); target.validityBuffer.retain(1); } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ target.allocateValidityBuffer(byteSizeTarget); for (int i = 0; i < byteSizeTarget - 1; i++) { @@ -742,16 +743,15 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, target.validityBuffer.setByte(i, (b1 + b2)); } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); @@ -1157,13 +1157,13 @@ protected final void fillHoles(int index) { } protected final void setBytes(int index, byte[] value, int start, int length) { - /* end offset of current last element in the vector. this will - * be the start offset of new element we are trying to store. - */ + /* end offset of current last element in the vector. this will + * be the start offset of new element we are trying to store. + */ final int startOffset = getstartOffset(index); - /* set new end offset */ + /* set new end offset */ offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); - /* store the var length data in value buffer */ + /* store the var length data in value buffer */ valueBuffer.setBytes(startOffset, value, start, length); } @@ -1172,26 +1172,26 @@ protected final int getstartOffset(int index) { } protected final void handleSafe(int index, int dataLength) { - /* - * IMPORTANT: - * value buffer for variable length vectors moves independent - * of the companion validity and offset buffers. This is in - * contrast to what we have for fixed width vectors. - * - * Here there is no concept of getValueCapacity() in the - * data stream. getValueCapacity() is applicable only to validity - * and offset buffers. - * - * So even though we may have setup an initial capacity of 1024 - * elements in the vector, it is quite possible - * that we need to reAlloc() the data buffer when we are setting - * the 5th element in the vector simply because previous - * variable length elements have exhausted the buffer capacity. - * However, we really don't need to reAlloc() validity and - * offset buffers until we try to set the 1025th element - * This is why we do a separate check for safe methods to - * determine which buffer needs reallocation. - */ + /* + * IMPORTANT: + * value buffer for variable length vectors moves independent + * of the companion validity and offset buffers. This is in + * contrast to what we have for fixed width vectors. + * + * Here there is no concept of getValueCapacity() in the + * data stream. getValueCapacity() is applicable only to validity + * and offset buffers. + * + * So even though we may have setup an initial capacity of 1024 + * elements in the vector, it is quite possible + * that we need to reAlloc() the data buffer when we are setting + * the 5th element in the vector simply because previous + * variable length elements have exhausted the buffer capacity. + * However, we really don't need to reAlloc() validity and + * offset buffers until we try to set the 1025th element + * This is why we do a separate check for safe methods to + * determine which buffer needs reallocation. + */ while (index >= getValueCapacity()) { reallocValidityAndOffsetBuffers(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index cc20beda3da..23252ca697b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -143,16 +143,16 @@ public static int getNullCount(final ArrowBuf validityBuffer, final int valueCou for (int i = 0; i < sizeInBytes; ++i) { final byte byteValue = validityBuffer.getByte(i); - /* Java uses two's complement binary representation, hence 11111111_b which is -1 - * when converted to Int will have 32bits set to 1. Masking the MSB and then - * adding it back solves the issue. - */ + /* Java uses two's complement binary representation, hence 11111111_b which is -1 + * when converted to Int will have 32bits set to 1. Masking the MSB and then + * adding it back solves the issue. + */ count += Integer.bitCount(byteValue & 0x7F) - (byteValue >> 7); } int nullCount = (sizeInBytes * 8) - count; - /* if the valueCount is not a multiple of 8, - * the bits on the right were counted as null bits. - */ + /* if the valueCount is not a multiple of 8, + * the bits on the right were counted as null bits. + */ int remainder = valueCount % 8; nullCount -= remainder == 0 ? 0 : 8 - remainder; return nullCount; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index d626c9bdbf5..afe86a692c3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -107,6 +107,13 @@ public List getChildrenFromFields() { return singletonList(getDataVector()); } + /** + * Load the buffers of this vector with provided source buffers. + * The caller manages the source buffers and populates them before invoking + * this method. + * @param fieldNode the fieldNode indicating the value count + * @param ownBuffers the buffers for this Field (own buffers only, children not included) + */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 2) { @@ -128,6 +135,10 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers valueCount = fieldNode.getLength(); } + /** + * Get the buffers belonging to this vector + * @return the inner buffers. + */ @Override public List getFieldBuffers() { List result = new ArrayList<>(2); @@ -138,6 +149,9 @@ public List getFieldBuffers() { return result; } + /** + * Set the reader and writer indexes for the inner buffers. + */ private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); @@ -160,6 +174,9 @@ public UnionListWriter getWriter() { return new UnionListWriter(this); } + /** + * Same as {@link #allocateNewSafe()}. + */ @Override public void allocateNew() throws OutOfMemoryException { if (!allocateNewSafe()) { @@ -167,6 +184,13 @@ public void allocateNew() throws OutOfMemoryException { } } + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. + * + * @return false if memory allocation fails, true otherwise. + */ public boolean allocateNewSafe() { boolean success = false; try { @@ -193,6 +217,10 @@ private void allocateValidityBuffer(final long size) { validityBuffer.setZero(0, validityBuffer.capacity()); } + /** + * Resize the vector to increase the capacity. The internal behavior is to + * double the current value capacity. + */ @Override public void reAlloc() { /* reallocate the validity buffer */ @@ -230,10 +258,25 @@ private void reallocValidityBuffer() { validityAllocationSizeInBytes = (int) newAllocationSize; } + /** + * Same as {@link #copyFrom(int, int, ListVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + */ public void copyFromSafe(int inIndex, int outIndex, ListVector from) { copyFrom(inIndex, outIndex, from); } + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector + * @param inIndex position to copy from in source vector + * @param outIndex position to copy to in this vector + * @param from source vector + */ public void copyFrom(int inIndex, int outIndex, ListVector from) { FieldReader in = from.getReader(); in.setPosition(inIndex); @@ -242,6 +285,10 @@ public void copyFrom(int inIndex, int outIndex, ListVector from) { ComplexCopier.copy(in, out); } + /** + * Get the inner data vector for this list vector + * @return data vector + */ @Override public FieldVector getDataVector() { return vector; @@ -310,6 +357,11 @@ public TransferImpl(ListVector to) { dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); } + /** + * Transfer this vector'data to another vector. The memory associated + * with this vector is transferred to the allocator of target vector + * for accounting and management purposes. + */ @Override public void transfer() { to.clear(); @@ -323,6 +375,12 @@ public void transfer() { clear(); } + /** + * Slice this vector at desired index and length and transfer the + * corresponding data to the target vector. + * @param startIndex start position of the split in source vector. + * @param length length of the split. + */ @Override public void splitAndTransfer(int startIndex, int length) { final int startPoint = offsetBuffer.getInt(startIndex * OFFSET_WIDTH); @@ -423,6 +481,11 @@ public AddOrGetResult addOrGetVector(FieldType fieldT return result; } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { if (valueCount == 0) { @@ -450,6 +513,17 @@ public void clear() { lastSet = 0; } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { setReaderAndWriterIndex(); @@ -481,6 +555,11 @@ public UnionVector promoteToUnion() { return vector; } + /** + * Get the element in the list vector at a particular index + * @param index position of the element + * @return Object at given position + */ @Override public Object getObject(int index) { if (isSet(index) == 0) { @@ -497,11 +576,23 @@ public Object getObject(int index) { return vals; } + /** + * Check if element at given index is null. + * + * @param index position of element + * @return true if element at given index is null, false otherwise + */ @Override public boolean isNull(int index) { return (isSet(index) == 0); } + /** + * Same as {@link #isNull(int)}. + * + * @param index position of element + * @return 1 if element at given index is not null, 0 otherwise + */ public int isSet(int index) { final int byteIndex = index >> 3; final byte b = validityBuffer.getByte(byteIndex); @@ -509,11 +600,20 @@ public int isSet(int index) { return Long.bitCount(b & (1L << bitIndex)); } + /** + * Get the number of elements that are null in the vector + * + * @return the number of null elements. + */ @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity() { return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); @@ -536,6 +636,11 @@ public void setNotNull(int index) { lastSet = index + 1; } + /** + * Start a new value in the list vector + * + * @param index index of the value to start + */ @Override public int startNewValue(int index) { while (index >= getValidityAndOffsetValueCapacity()) { @@ -561,6 +666,11 @@ public void endValue(int index, int size) { offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); } + /** + * Sets the value count for the vector + * + * @param valueCount value count + */ @Override public void setValueCount(int valueCount) { this.valueCount = valueCount; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 533efc9063c..f95302f55f8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -238,16 +238,35 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, Nullable } } + /** + * Get the value capacity of the internal validity buffer. + * @return number of elements that validity buffer can hold + */ private int getValidityBufferValueCapacity() { return (int) (validityBuffer.capacity() * 8L); } + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ @Override public int getValueCapacity() { return Math.min(getValidityBufferValueCapacity(), super.getValueCapacity()); } + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ @Override public ArrowBuf[] getBuffers(boolean clear) { setReaderAndWriterIndex(); @@ -268,23 +287,37 @@ public ArrowBuf[] getBuffers(boolean clear) { return buffers; } + /** + * Close the vector and release the associated buffers. + */ @Override public void close() { clearValidityBuffer(); super.close(); } + /** + * Same as {@link #close()} + */ @Override public void clear() { clearValidityBuffer(); super.clear(); } + /** + * Release the validity buffer + */ private void clearValidityBuffer() { validityBuffer.release(); validityBuffer = allocator.getEmpty(); } + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ @Override public int getBufferSize() { if (valueCount == 0) { @@ -294,6 +327,12 @@ public int getBufferSize() { BitVectorHelper.getValidityBufferSize(valueCount); } + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ @Override public int getBufferSizeFor(final int valueCount) { if (valueCount == 0) {