diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java index 7c652902eb8..5357f9b8a9d 100644 --- a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java +++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java @@ -23,7 +23,7 @@ <#if mode == "Single"> <#assign containerClass = "MapVector" /> <#else> -<#assign containerClass = "MapVector" /> +<#assign containerClass = "NullableMapVector" /> <#include "/@includes/license.ftl" /> diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index a5ac1b71704..ac59e59b308 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -21,9 +21,9 @@ <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}MapWriter.java" /> <#assign index = "idx()"> <#if mode == "Single"> -<#assign containerClass = "NonNullableMapVector" /> -<#else> <#assign containerClass = "MapVector" /> +<#else> +<#assign containerClass = "NullableMapVector" /> <#include "/@includes/license.ftl" /> @@ -51,7 +51,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter { private final Map fields = Maps.newHashMap(); public ${mode}MapWriter(${containerClass} container) { <#if mode == "Single"> - if (container instanceof MapVector) { + if (container instanceof NullableMapVector) { throw new IllegalArgumentException("Invalid container: " + container); } @@ -124,7 +124,7 @@ public MapWriter map(String name) { FieldWriter writer = fields.get(finalName); if(writer == null){ int vectorCount=container.size(); - MapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), MapVector.class); + NullableMapVector vector = container.addOrGet(name, FieldType.nullable(MinorType.MAP.getType()), NullableMapVector.class); writer = new PromotableWriter(vector, container, getNullableMapWriterFactory()); if(vectorCount != container.size()) { writer.allocate(); diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 60cd24dcc44..9377bd0da21 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -48,8 +48,8 @@ /** - * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each - * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods, + * A vector which can hold values of different types. It does so by using a NullableMapVector which contains a vector for each + * primitive type that is stored. NullableMapVector is used in order to take advantage of its serialization/deserialization methods, * as well as the addOrGet method. * * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup @@ -65,7 +65,7 @@ public class UnionVector implements FieldVector { MapVector internalMap; protected ArrowBuf typeBuffer; - private MapVector mapVector; + private NullableMapVector mapVector; private ListVector listVector; private FieldReader reader; @@ -174,10 +174,10 @@ public long getOffsetBufferAddress() { @Override public ArrowBuf getOffsetBuffer() { throw new UnsupportedOperationException(); } - public MapVector getMap() { + public NullableMapVector getMap() { if (mapVector == null) { int vectorCount = internalMap.size(); - mapVector = addOrGet(MinorType.MAP, MapVector.class); + mapVector = addOrGet(MinorType.MAP, NullableMapVector.class); if (internalMap.size() > vectorCount) { mapVector.allocateNew(); if (callBack != null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 98820a2367b..fbadb35d755 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -24,7 +24,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index 5ded1cc6f87..db0ff86df47 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -100,8 +100,8 @@ protected boolean supportsDirectRead() { // return the child vector's ordinal in the composite container public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name); - public MapVector addOrGetMap(String name) { - return addOrGet(name, FieldType.nullable(new Struct()), MapVector.class); + public NullableMapVector addOrGetMap(String name) { + return addOrGet(name, FieldType.nullable(new Struct()), NullableMapVector.class); } public ListVector addOrGetList(String name) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 72cc29f30ca..e13084588ce 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,480 +21,322 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; +import java.util.Collection; +import java.util.Iterator; import java.util.List; +import java.util.Map; -import com.google.common.collect.ObjectArrays; +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; import io.netty.buffer.ArrowBuf; -import org.apache.arrow.memory.BaseAllocator; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; -import org.apache.arrow.vector.complex.impl.NullableMapWriter; +import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.ArrowType.Struct; -import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.TransferPair; -public class MapVector extends NonNullableMapVector implements FieldVector { +public class MapVector extends AbstractMapVector { + //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); public static MapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = FieldType.nullable(Struct.INSTANCE); + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); return new MapVector(name, allocator, fieldType, null); } - private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); - private final NullableMapWriter writer = new NullableMapWriter(this); - - protected ArrowBuf validityBuffer; - private int validityAllocationSizeInBytes; + private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); + protected final FieldType fieldType; + public int valueCount; // deprecated, use FieldType or static constructor instead @Deprecated public MapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); + this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); } - // deprecated, use FieldType or static constructor instead - @Deprecated - public MapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); + public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, callBack); + this.fieldType = checkNotNull(fieldType); + this.valueCount = 0; } - public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, checkNotNull(allocator), fieldType, callBack); - this.validityBuffer = allocator.getEmpty(); - this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + @Override + public FieldReader getReader() { + return reader; + } + + transient private MapTransferPair ephPair; + + public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) { + if (ephPair == null || ephPair.from != from) { + ephPair = (MapTransferPair) from.makeTransferPair(this); + } + ephPair.copyValueSafe(fromIndex, thisIndex); } @Override - public Field getField() { - Field f = super.getField(); - FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); - return new Field(f.getName(), type, f.getChildren()); + protected boolean supportsDirectRead() { + return true; + } + + public Iterator fieldNameIterator() { + return getChildFieldNames().iterator(); } @Override - public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { - if (ownBuffers.size() != 1) { - throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + public void setInitialCapacity(int numRecords) { + for (final ValueVector v : (Iterable) this) { + v.setInitialCapacity(numRecords); } + } - ArrowBuf bitBuffer = ownBuffers.get(0); + @Override + public int getBufferSize() { + if (valueCount == 0 || size() == 0) { + return 0; + } + long buffer = 0; + for (final ValueVector v : (Iterable) this) { + buffer += v.getBufferSize(); + } - validityBuffer.release(); - validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); - valueCount = fieldNode.getLength(); - validityAllocationSizeInBytes = validityBuffer.capacity(); + return (int) buffer; } @Override - public List getFieldBuffers() { - List result = new ArrayList<>(1); - setReaderAndWriterIndex(); - result.add(validityBuffer); + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } - return result; - } + long bufferSize = 0; + for (final ValueVector v : (Iterable) this) { + bufferSize += v.getBufferSizeFor(valueCount); + } - private void setReaderAndWriterIndex() { - validityBuffer.readerIndex(0); - validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + return (int) bufferSize; } @Override - @Deprecated - public List getFieldInnerVectors() { - throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + public ArrowBuf getValidityBuffer() { + throw new UnsupportedOperationException(); } @Override - public NullableMapReaderImpl getReader() { - return reader; + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); } - public NullableMapWriter getWriter() { - return writer; + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); } @Override public TransferPair getTransferPair(BufferAllocator allocator) { - return new NullableMapTransferPair(this, new MapVector(name, allocator, fieldType, null), false); + return getTransferPair(name, allocator, null); } @Override - public TransferPair makeTransferPair(ValueVector to) { - return new NullableMapTransferPair(this, (MapVector) to, true); + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, null), false); + public TransferPair makeTransferPair(ValueVector to) { + return new MapTransferPair(this, (MapVector) to); } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new NullableMapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); } - protected class NullableMapTransferPair extends MapTransferPair { - - private MapVector target; + protected static class MapTransferPair implements TransferPair { + private final TransferPair[] pairs; + private final MapVector from; + private final MapVector to; - protected NullableMapTransferPair(MapVector from, MapVector to, boolean allocate) { - super(from, to, allocate); - this.target = to; + public MapTransferPair(MapVector from, MapVector to) { + this(from, to, true); } - @Override - public void transfer() { - target.clear(); - target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; - super.transfer(); - clear(); + protected MapTransferPair(MapVector from, MapVector to, boolean allocate) { + this.from = from; + this.to = to; + this.pairs = new TransferPair[from.size()]; + this.to.ephPair = null; + + int i = 0; + FieldVector vector; + for (String child : from.getChildFieldNames()) { + int preSize = to.size(); + vector = from.getChild(child); + if (vector == null) { + continue; + } + //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, + // the child fields may be nested fields of the top level child. For example if the structure + // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. + // But the children member of a Materialized field is a HashSet. If the fields are added in the + // children HashSet, and the hashCode of the Materialized field includes the hash code of the + // children, the hashCode value of oa changes *after* the field has been added to the HashSet. + // (This is similar to what happens in ScanBatch where the children cannot be added till they are + // read). To take care of this, we ensure that the hashCode of the MaterializedField does not + // include the hashCode of the children but is based only on MaterializedField$key. + final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); + if (allocate && to.size() != preSize) { + newVector.allocateNew(); + } + pairs[i++] = vector.makeTransferPair(newVector); + } } @Override - public void copyValueSafe(int fromIndex, int toIndex) { - while (toIndex >= target.getValidityBufferValueCapacity()) { - target.reallocValidityBuffer(); + public void transfer() { + for (final TransferPair p : pairs) { + p.transfer(); } - BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); - super.copyValueSafe(fromIndex, toIndex); + to.valueCount = from.valueCount; + from.clear(); } @Override - public void splitAndTransfer(int startIndex, int length) { - target.clear(); - splitAndTransferValidityBuffer(startIndex, length, target); - super.splitAndTransfer(startIndex, length); + public ValueVector getTo() { + return to; } - } - /* - * transfer the validity. - */ - private void splitAndTransferValidityBuffer(int startIndex, int length, MapVector target) { - assert startIndex + length <= valueCount; - int firstByteSource = BitVectorHelper.byteIndex(startIndex); - int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); - int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); - int offset = startIndex % 8; - - if (length > 0) { - if (offset == 0) { - // slice - if (target.validityBuffer != null) { - target.validityBuffer.release(); - } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.retain(1); - } else { - /* Copy data - * When the first bit starts from the middle of a byte (offset != 0), - * copy data from src BitVector. - * Each byte in the target is composed by a part in i-th byte, - * another part in (i+1)-th byte. - */ - target.allocateValidityBuffer(byteSizeTarget); - - for (int i = 0; i < byteSizeTarget - 1; i++) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); - - target.validityBuffer.setByte(i, (b1 + b2)); - } - - /* Copying the last piece is done in the following manner: - * if the source vector has 1 or more bytes remaining, we copy - * the last piece as a byte formed by shifting data - * from the current byte and the next byte. - * - * if the source vector has no more bytes remaining - * (we are at the last byte), we copy the last piece as a byte - * by shifting data from the current byte. - */ - if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, - firstByteSource + byteSizeTarget, offset); - - target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); - } else { - byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, - firstByteSource + byteSizeTarget - 1, offset); - target.validityBuffer.setByte(byteSizeTarget - 1, b1); - } + @Override + public void copyValueSafe(int from, int to) { + for (TransferPair p : pairs) { + p.copyValueSafe(from, to); } } - } - - /** - * Get the value capacity of the internal validity buffer. - * @return number of elements that validity buffer can hold - */ - private int getValidityBufferValueCapacity() { - return (int) (validityBuffer.capacity() * 8L); - } - - /** - * Get the current value capacity for the vector - * @return number of elements that vector can hold. - */ - @Override - public int getValueCapacity() { - return Math.min(getValidityBufferValueCapacity(), - super.getValueCapacity()); - } - /** - * Return the underlying buffers associated with this vector. Note that this doesn't - * impact the reference counts for this buffer so it only should be used for in-context - * access. Also note that this buffer changes regularly thus - * external classes shouldn't hold a reference to it (unless they change it). - * - * @param clear Whether to clear vector before returning; the buffers will still be refcounted - * but the returned array will be the only reference to them - * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this - * vector instance. - */ - @Override - public ArrowBuf[] getBuffers(boolean clear) { - setReaderAndWriterIndex(); - final ArrowBuf[] buffers; - if (getBufferSize() == 0) { - buffers = new ArrowBuf[0]; - } else { - buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), - ArrowBuf.class); - } - if (clear) { - for (ArrowBuf buffer : buffers) { - buffer.retain(); + @Override + public void splitAndTransfer(int startIndex, int length) { + for (TransferPair p : pairs) { + p.splitAndTransfer(startIndex, length); } - clear(); + to.setValueCount(length); } - - return buffers; } - /** - * Close the vector and release the associated buffers. - */ @Override - public void close() { - clearValidityBuffer(); - super.close(); - } - - /** - * Same as {@link #close()} - */ - @Override - public void clear() { - clearValidityBuffer(); - super.clear(); - } - - /** - * Release the validity buffer - */ - private void clearValidityBuffer() { - validityBuffer.release(); - validityBuffer = allocator.getEmpty(); - } - - /** - * Get the size (number of bytes) of underlying buffers used by this - * vector - * @return size of underlying buffers. - */ - @Override - public int getBufferSize() { - if (valueCount == 0) { + public int getValueCapacity() { + if (size() == 0) { return 0; } - return super.getBufferSize() + - BitVectorHelper.getValidityBufferSize(valueCount); - } - /** - * Get the potential buffer size for a particular number of records. - * @param valueCount desired number of elements in the vector - * @return estimated size of underlying buffers if the vector holds - * a given number of elements - */ - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - return super.getBufferSizeFor(valueCount) - + BitVectorHelper.getValidityBufferSize(valueCount); - } + final Ordering natural = new Ordering() { + @Override + public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { + return Ints.compare( + checkNotNull(left).getValueCapacity(), + checkNotNull(right).getValueCapacity() + ); + } + }; - @Override - public void setInitialCapacity(int numRecords) { - validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); - super.setInitialCapacity(numRecords); + return natural.min(getChildren()).getValueCapacity(); } @Override - public boolean allocateNewSafe() { - /* Boolean to keep track if all the memory allocations were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated - */ - boolean success = false; - try { - clear(); - allocateValidityBuffer(validityAllocationSizeInBytes); - success = super.allocateNewSafe(); - } finally { - if (!success) { - clear(); - return false; + public Object getObject(int index) { + Map vv = new JsonStringHashMap<>(); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null && index < v.getValueCount()) { + Object value = v.getObject(index); + if (value != null) { + vv.put(child, value); + } } } - return true; - } - - private void allocateValidityBuffer(final long size) { - final int curSize = (int) size; - validityBuffer = allocator.buffer(curSize); - validityBuffer.readerIndex(0); - validityAllocationSizeInBytes = curSize; - validityBuffer.setZero(0, validityBuffer.capacity()); + return vv; } @Override - public void reAlloc() { - /* reallocate the validity buffer */ - reallocValidityBuffer(); - super.reAlloc(); - } - - private void reallocValidityBuffer() { - final int currentBufferCapacity = validityBuffer.capacity(); - long baseSize = validityAllocationSizeInBytes; - - if (baseSize < (long) currentBufferCapacity) { - baseSize = (long) currentBufferCapacity; - } - - long newAllocationSize = baseSize * 2L; - newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); - - if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { - throw new OversizedAllocationException("Unable to expand the buffer"); - } - - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); - newBuf.setZero(0, newBuf.capacity()); - newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); - validityBuffer.release(1); - validityBuffer = newBuf; - validityAllocationSizeInBytes = (int) newAllocationSize; - } - + public boolean isNull(int index) { return false; } @Override - public long getValidityBufferAddress() { - return validityBuffer.memoryAddress(); - } + public int getNullCount() { return 0; } - @Override - public long getDataBufferAddress() { - throw new UnsupportedOperationException(); + public void get(int index, ComplexHolder holder) { + reader.setPosition(index); + holder.reader = reader; } @Override - public long getOffsetBufferAddress() { - throw new UnsupportedOperationException(); + public int getValueCount() { + return valueCount; } - @Override - public ArrowBuf getValidityBuffer() { - return validityBuffer; - } + public ValueVector getVectorById(int id) { + return getChildByOrdinal(id); +} @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); + public void setValueCount(int valueCount) { + for (final ValueVector v : getChildren()) { + v.setValueCount(valueCount); + } + MapVector.this.valueCount = valueCount; } @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); + public void clear() { + for (final ValueVector v : getChildren()) { + v.clear(); + } + valueCount = 0; } @Override - public Object getObject(int index) { - if (isSet(index) == 0) { - return null; - } else { - return super.getObject(index); + public Field getField() { + List children = new ArrayList<>(); + for (ValueVector child : getChildren()) { + children.add(child.getField()); } + return new Field(name, fieldType, children); } @Override - public void get(int index, ComplexHolder holder) { - holder.isSet = isSet(index); - super.get(index, holder); + public MinorType getMinorType() { + return MinorType.MAP; } - public int getNullCount() { - return BitVectorHelper.getNullCount(validityBuffer, valueCount); - } - - public boolean isNull(int index) { - return isSet(index) == 0; - } - - public int isSet(int index) { - final int byteIndex = index >> 3; - final byte b = validityBuffer.getByte(byteIndex); - final int bitIndex = index & 7; - return Long.bitCount(b & (1L << bitIndex)); - } - - public void setIndexDefined(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); + @Override + public void close() { + final Collection vectors = getChildren(); + for (final FieldVector v : vectors) { + v.close(); } - BitVectorHelper.setValidityBitToOne(validityBuffer, index); - } + vectors.clear(); - public void setNull(int index) { - while (index >= getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); - } - BitVectorHelper.setValidityBit(validityBuffer, index, 0); + valueCount = 0; + + super.close(); } - @Override - public void setValueCount(int valueCount) { - assert valueCount >= 0; - while (valueCount > getValidityBufferValueCapacity()) { - /* realloc the inner buffers if needed */ - reallocValidityBuffer(); + public void initializeChildrenFromFields(List children) { + for (Field field : children) { + FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); + vector.initializeChildrenFromFields(field.getChildren()); } - super.setValueCount(valueCount); - this.valueCount = valueCount; } - public void reset() { - valueCount = 0; + public List getChildrenFromFields() { + return getChildren(); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java deleted file mode 100644 index 1cd7b70d7bf..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableMapVector.java +++ /dev/null @@ -1,342 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.complex; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; - -import io.netty.buffer.ArrowBuf; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.*; -import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; -import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.ComplexHolder; -import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; -import org.apache.arrow.vector.util.CallBack; -import org.apache.arrow.vector.util.JsonStringHashMap; -import org.apache.arrow.vector.util.TransferPair; - -public class NonNullableMapVector extends AbstractMapVector { - //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NonNullableMapVector.class); - - public static NonNullableMapVector empty(String name, BufferAllocator allocator) { - FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); - return new NonNullableMapVector(name, allocator, fieldType, null); - } - - private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); - protected final FieldType fieldType; - public int valueCount; - - // deprecated, use FieldType or static constructor instead - @Deprecated - public NonNullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); - } - - public NonNullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { - super(name, allocator, callBack); - this.fieldType = checkNotNull(fieldType); - this.valueCount = 0; - } - - @Override - public FieldReader getReader() { - return reader; - } - - transient private MapTransferPair ephPair; - - public void copyFromSafe(int fromIndex, int thisIndex, NonNullableMapVector from) { - if (ephPair == null || ephPair.from != from) { - ephPair = (MapTransferPair) from.makeTransferPair(this); - } - ephPair.copyValueSafe(fromIndex, thisIndex); - } - - @Override - protected boolean supportsDirectRead() { - return true; - } - - public Iterator fieldNameIterator() { - return getChildFieldNames().iterator(); - } - - @Override - public void setInitialCapacity(int numRecords) { - for (final ValueVector v : (Iterable) this) { - v.setInitialCapacity(numRecords); - } - } - - @Override - public int getBufferSize() { - if (valueCount == 0 || size() == 0) { - return 0; - } - long buffer = 0; - for (final ValueVector v : (Iterable) this) { - buffer += v.getBufferSize(); - } - - return (int) buffer; - } - - @Override - public int getBufferSizeFor(final int valueCount) { - if (valueCount == 0) { - return 0; - } - - long bufferSize = 0; - for (final ValueVector v : (Iterable) this) { - bufferSize += v.getBufferSizeFor(valueCount); - } - - return (int) bufferSize; - } - - @Override - public ArrowBuf getValidityBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getDataBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public ArrowBuf getOffsetBuffer() { - throw new UnsupportedOperationException(); - } - - @Override - public TransferPair getTransferPair(BufferAllocator allocator) { - return getTransferPair(name, allocator, null); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new MapTransferPair(this, new NonNullableMapVector(name, allocator, fieldType, callBack), false); - } - - @Override - public TransferPair makeTransferPair(ValueVector to) { - return new MapTransferPair(this, (NonNullableMapVector) to); - } - - @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new MapTransferPair(this, new NonNullableMapVector(ref, allocator, fieldType, callBack), false); - } - - protected static class MapTransferPair implements TransferPair { - private final TransferPair[] pairs; - private final NonNullableMapVector from; - private final NonNullableMapVector to; - - public MapTransferPair(NonNullableMapVector from, NonNullableMapVector to) { - this(from, to, true); - } - - protected MapTransferPair(NonNullableMapVector from, NonNullableMapVector to, boolean allocate) { - this.from = from; - this.to = to; - this.pairs = new TransferPair[from.size()]; - this.to.ephPair = null; - - int i = 0; - FieldVector vector; - for (String child : from.getChildFieldNames()) { - int preSize = to.size(); - vector = from.getChild(child); - if (vector == null) { - continue; - } - //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector, - // the child fields may be nested fields of the top level child. For example if the structure - // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab. - // But the children member of a Materialized field is a HashSet. If the fields are added in the - // children HashSet, and the hashCode of the Materialized field includes the hash code of the - // children, the hashCode value of oa changes *after* the field has been added to the HashSet. - // (This is similar to what happens in ScanBatch where the children cannot be added till they are - // read). To take care of this, we ensure that the hashCode of the MaterializedField does not - // include the hashCode of the children but is based only on MaterializedField$key. - final FieldVector newVector = to.addOrGet(child, vector.getField().getFieldType(), vector.getClass()); - if (allocate && to.size() != preSize) { - newVector.allocateNew(); - } - pairs[i++] = vector.makeTransferPair(newVector); - } - } - - @Override - public void transfer() { - for (final TransferPair p : pairs) { - p.transfer(); - } - to.valueCount = from.valueCount; - from.clear(); - } - - @Override - public ValueVector getTo() { - return to; - } - - @Override - public void copyValueSafe(int from, int to) { - for (TransferPair p : pairs) { - p.copyValueSafe(from, to); - } - } - - @Override - public void splitAndTransfer(int startIndex, int length) { - for (TransferPair p : pairs) { - p.splitAndTransfer(startIndex, length); - } - to.setValueCount(length); - } - } - - @Override - public int getValueCapacity() { - if (size() == 0) { - return 0; - } - - final Ordering natural = new Ordering() { - @Override - public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { - return Ints.compare( - checkNotNull(left).getValueCapacity(), - checkNotNull(right).getValueCapacity() - ); - } - }; - - return natural.min(getChildren()).getValueCapacity(); - } - - @Override - public Object getObject(int index) { - Map vv = new JsonStringHashMap<>(); - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null && index < v.getValueCount()) { - Object value = v.getObject(index); - if (value != null) { - vv.put(child, value); - } - } - } - return vv; - } - - @Override - public boolean isNull(int index) { return false; } - @Override - public int getNullCount() { return 0; } - - public void get(int index, ComplexHolder holder) { - reader.setPosition(index); - holder.reader = reader; - } - - @Override - public int getValueCount() { - return valueCount; - } - - public ValueVector getVectorById(int id) { - return getChildByOrdinal(id); -} - - @Override - public void setValueCount(int valueCount) { - for (final ValueVector v : getChildren()) { - v.setValueCount(valueCount); - } - NonNullableMapVector.this.valueCount = valueCount; - } - - @Override - public void clear() { - for (final ValueVector v : getChildren()) { - v.clear(); - } - valueCount = 0; - } - - @Override - public Field getField() { - List children = new ArrayList<>(); - for (ValueVector child : getChildren()) { - children.add(child.getField()); - } - return new Field(name, fieldType, children); - } - - @Override - public MinorType getMinorType() { - return MinorType.MAP; - } - - @Override - public void close() { - final Collection vectors = getChildren(); - for (final FieldVector v : vectors) { - v.close(); - } - vectors.clear(); - - valueCount = 0; - - super.close(); - } - - public void initializeChildrenFromFields(List children) { - for (Field field : children) { - FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); - vector.initializeChildrenFromFields(field.getChildren()); - } - } - - public List getChildrenFromFields() { - return getChildren(); - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java new file mode 100644 index 00000000000..d887b730646 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -0,0 +1,500 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; +import org.apache.arrow.memory.BaseAllocator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; +import org.apache.arrow.vector.complex.impl.NullableMapWriter; +import org.apache.arrow.vector.holders.ComplexHolder; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.OversizedAllocationException; +import org.apache.arrow.vector.util.TransferPair; + +public class NullableMapVector extends MapVector implements FieldVector { + + public static NullableMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(Struct.INSTANCE); + return new NullableMapVector(name, allocator, fieldType, null); + } + + private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); + private final NullableMapWriter writer = new NullableMapWriter(this); + + protected ArrowBuf validityBuffer; + private int validityAllocationSizeInBytes; + + // deprecated, use FieldType or static constructor instead + @Deprecated + public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); + } + + // deprecated, use FieldType or static constructor instead + @Deprecated + public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); + } + + public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, checkNotNull(allocator), fieldType, callBack); + this.validityBuffer = allocator.getEmpty(); + this.validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION); + } + + @Override + public Field getField() { + Field f = super.getField(); + FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); + return new Field(f.getName(), type, f.getChildren()); + } + + @Override + public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { + if (ownBuffers.size() != 1) { + throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); + } + + ArrowBuf bitBuffer = ownBuffers.get(0); + + validityBuffer.release(); + validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); + valueCount = fieldNode.getLength(); + validityAllocationSizeInBytes = validityBuffer.capacity(); + } + + @Override + public List getFieldBuffers() { + List result = new ArrayList<>(1); + setReaderAndWriterIndex(); + result.add(validityBuffer); + + return result; + } + + private void setReaderAndWriterIndex() { + validityBuffer.readerIndex(0); + validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSize(valueCount)); + } + + @Override + @Deprecated + public List getFieldInnerVectors() { + throw new UnsupportedOperationException("There are no inner vectors. Use getFieldBuffers"); + } + + @Override + public NullableMapReaderImpl getReader() { + return reader; + } + + public NullableMapWriter getWriter() { + return writer; + } + + @Override + public TransferPair getTransferPair(BufferAllocator allocator) { + return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false); + } + + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new NullableMapTransferPair(this, (NullableMapVector) to, true); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false); + } + + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false); + } + + protected class NullableMapTransferPair extends MapTransferPair { + + private NullableMapVector target; + + protected NullableMapTransferPair(NullableMapVector from, NullableMapVector to, boolean allocate) { + super(from, to, allocate); + this.target = to; + } + + @Override + public void transfer() { + target.clear(); + target.validityBuffer = validityBuffer.transferOwnership(target.allocator).buffer; + super.transfer(); + clear(); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + while (toIndex >= target.getValidityBufferValueCapacity()) { + target.reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(target.validityBuffer, toIndex, isSet(fromIndex)); + super.copyValueSafe(fromIndex, toIndex); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + target.clear(); + splitAndTransferValidityBuffer(startIndex, length, target); + super.splitAndTransfer(startIndex, length); + } + } + + /* + * transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, NullableMapVector target) { + assert startIndex + length <= valueCount; + int firstByteSource = BitVectorHelper.byteIndex(startIndex); + int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); + int offset = startIndex % 8; + + if (length > 0) { + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.release(); + } + target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer.retain(1); + } else { + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + } + } + + /** + * Get the value capacity of the internal validity buffer. + * @return number of elements that validity buffer can hold + */ + private int getValidityBufferValueCapacity() { + return (int) (validityBuffer.capacity() * 8L); + } + + /** + * Get the current value capacity for the vector + * @return number of elements that vector can hold. + */ + @Override + public int getValueCapacity() { + return Math.min(getValidityBufferValueCapacity(), + super.getValueCapacity()); + } + + /** + * Return the underlying buffers associated with this vector. Note that this doesn't + * impact the reference counts for this buffer so it only should be used for in-context + * access. Also note that this buffer changes regularly thus + * external classes shouldn't hold a reference to it (unless they change it). + * + * @param clear Whether to clear vector before returning; the buffers will still be refcounted + * but the returned array will be the only reference to them + * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this + * vector instance. + */ + @Override + public ArrowBuf[] getBuffers(boolean clear) { + setReaderAndWriterIndex(); + final ArrowBuf[] buffers; + if (getBufferSize() == 0) { + buffers = new ArrowBuf[0]; + } else { + buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false), + ArrowBuf.class); + } + if (clear) { + for (ArrowBuf buffer : buffers) { + buffer.retain(); + } + clear(); + } + + return buffers; + } + + /** + * Close the vector and release the associated buffers. + */ + @Override + public void close() { + clearValidityBuffer(); + super.close(); + } + + /** + * Same as {@link #close()} + */ + @Override + public void clear() { + clearValidityBuffer(); + super.clear(); + } + + /** + * Release the validity buffer + */ + private void clearValidityBuffer() { + validityBuffer.release(); + validityBuffer = allocator.getEmpty(); + } + + /** + * Get the size (number of bytes) of underlying buffers used by this + * vector + * @return size of underlying buffers. + */ + @Override + public int getBufferSize() { + if (valueCount == 0) { + return 0; + } + return super.getBufferSize() + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + /** + * Get the potential buffer size for a particular number of records. + * @param valueCount desired number of elements in the vector + * @return estimated size of underlying buffers if the vector holds + * a given number of elements + */ + @Override + public int getBufferSizeFor(final int valueCount) { + if (valueCount == 0) { + return 0; + } + return super.getBufferSizeFor(valueCount) + + BitVectorHelper.getValidityBufferSize(valueCount); + } + + @Override + public void setInitialCapacity(int numRecords) { + validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); + super.setInitialCapacity(numRecords); + } + + @Override + public boolean allocateNewSafe() { + /* Boolean to keep track if all the memory allocations were successful + * Used in the case of composite vectors when we need to allocate multiple + * buffers for multiple vectors. If one of the allocations failed we need to + * clear all the memory that we allocated + */ + boolean success = false; + try { + clear(); + allocateValidityBuffer(validityAllocationSizeInBytes); + success = super.allocateNewSafe(); + } finally { + if (!success) { + clear(); + return false; + } + } + return true; + } + + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + validityAllocationSizeInBytes = curSize; + validityBuffer.setZero(0, validityBuffer.capacity()); + } + + @Override + public void reAlloc() { + /* reallocate the validity buffer */ + reallocValidityBuffer(); + super.reAlloc(); + } + + private void reallocValidityBuffer() { + final int currentBufferCapacity = validityBuffer.capacity(); + long baseSize = validityAllocationSizeInBytes; + + if (baseSize < (long) currentBufferCapacity) { + baseSize = (long) currentBufferCapacity; + } + + long newAllocationSize = baseSize * 2L; + newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize); + + if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { + throw new OversizedAllocationException("Unable to expand the buffer"); + } + + final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + newBuf.setZero(0, newBuf.capacity()); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + validityBuffer.release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + @Override + public long getValidityBufferAddress() { + return validityBuffer.memoryAddress(); + } + + @Override + public long getDataBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public long getOffsetBufferAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getValidityBuffer() { + return validityBuffer; + } + + @Override + public ArrowBuf getDataBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public ArrowBuf getOffsetBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + return super.getObject(index); + } + } + + @Override + public void get(int index, ComplexHolder holder) { + holder.isSet = isSet(index); + super.get(index, holder); + } + + public int getNullCount() { + return BitVectorHelper.getNullCount(validityBuffer, valueCount); + } + + public boolean isNull(int index) { + return isSet(index) == 0; + } + + public int isSet(int index) { + final int byteIndex = index >> 3; + final byte b = validityBuffer.getByte(byteIndex); + final int bitIndex = index & 7; + return Long.bitCount(b & (1L << bitIndex)); + } + + public void setIndexDefined(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + } + + public void setNull(int index) { + while (index >= getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + @Override + public void setValueCount(int valueCount) { + assert valueCount >= 0; + while (valueCount > getValidityBufferValueCapacity()) { + /* realloc the inner buffers if needed */ + reallocValidityBuffer(); + } + super.setValueCount(valueCount); + this.valueCount = valueCount; + } + + public void reset() { + valueCount = 0; + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java index 970b90ef510..a6960238b91 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java @@ -20,7 +20,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.StateTool; import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.types.pojo.Field; @@ -131,7 +131,7 @@ public MapWriter directMap() { switch (mode) { case INIT: - mapRoot = nullableMapWriterFactory.build((MapVector) container); + mapRoot = nullableMapWriterFactory.build((NullableMapVector) container); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -152,7 +152,7 @@ public MapWriter rootAsMap() { case INIT: // TODO allow dictionaries in complex types - MapVector map = container.addOrGetMap(name); + NullableMapVector map = container.addOrGetMap(name); mapRoot = nullableMapWriterFactory.build(map); mapRoot.setPosition(idx()); mode = Mode.MAP; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java index acf155af71d..06b0f4d4327 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapReaderImpl.java @@ -20,17 +20,17 @@ package org.apache.arrow.vector.complex.impl; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.Field; public class NullableMapReaderImpl extends SingleMapReaderImpl { - private MapVector nullableMapVector; + private NullableMapVector nullableMapVector; public NullableMapReaderImpl(MapVector vector) { - super((MapVector) vector); - this.nullableMapVector = (MapVector) vector; + super((NullableMapVector) vector); + this.nullableMapVector = (NullableMapVector) vector; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java index 06dbf784be0..d2dcb2374d0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.complex.impl; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; public class NullableMapWriterFactory { private final boolean caseSensitive; @@ -29,7 +29,7 @@ public NullableMapWriterFactory(boolean caseSensitive) { this.caseSensitive = caseSensitive; } - public NullableMapWriter build(MapVector container) { + public NullableMapWriter build(NullableMapVector container) { return this.caseSensitive ? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 9abd38d20ac..5bd439cac42 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -23,7 +23,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.AbstractMapVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -94,7 +94,7 @@ private void setWriter(ValueVector v) { type = v.getMinorType(); switch (type) { case MAP: - writer = nullableMapWriterFactory.build((MapVector) vector); + writer = nullableMapWriterFactory.build((NullableMapVector) vector); break; case LIST: writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java index 0341b622e0d..c77ca4e8f23 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleMapReaderImpl.java @@ -24,7 +24,6 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NonNullableMapVector; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -35,10 +34,10 @@ @SuppressWarnings("unused") public class SingleMapReaderImpl extends AbstractFieldReader { - private final NonNullableMapVector vector; + private final MapVector vector; private final Map fields = Maps.newHashMap(); - public SingleMapReaderImpl(NonNullableMapVector vector) { + public SingleMapReaderImpl(MapVector vector) { this.vector = vector; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 7834845aad8..3c5fd81d572 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -58,7 +58,7 @@ import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.BigIntWriterImpl; import org.apache.arrow.vector.complex.impl.BitWriterImpl; @@ -131,12 +131,12 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { MAP(Struct.INSTANCE) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { - return new MapVector(name, allocator, fieldType, schemaChangeCallback); + return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback); } @Override public FieldWriter getNewFieldWriter(ValueVector vector) { - return new NullableMapWriter((MapVector) vector); + return new NullableMapWriter((NullableMapVector) vector); } }, TINYINT(new Int(8, true)) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index e5682955bad..357df96aa2e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -22,7 +22,7 @@ import java.util.Map; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; @@ -50,7 +50,7 @@ public void testFieldMetadata() throws Exception { Map metadata = new HashMap<>(); metadata.put("k1", "v1"); FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata); - try (MapVector vector = new MapVector("map", allocator, type, null)) { + try (NullableMapVector vector = new NullableMapVector("map", allocator, type, null)) { Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 1b13c2ed919..f8edf8904c5 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -26,7 +26,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; @@ -121,7 +121,7 @@ public void testListType() { @Test public void testMapType() { - try (final MapVector vector = MapVector.empty("", allocator)) { + try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) { vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), IntVector.class); vector.setInitialCapacity(512); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 38b78424dcc..b0d6cf555e9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -25,7 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.DirtyRootAllocator; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -55,7 +55,7 @@ public void terminate() throws Exception { public void testPromoteToUnion() throws Exception { try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator); - final MapVector v = container.addOrGetMap("test"); + final NullableMapVector v = container.addOrGetMap("test"); final PromotableWriter writer = new PromotableWriter(v, container)) { container.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index bd8489eb20f..29d39aabe6b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -34,7 +34,7 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.NonNullableMapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl; @@ -830,7 +830,7 @@ public void complexCopierWithList() { rootWriter.end(); writer.setValueCount(1); - MapVector mapVector = (MapVector) parent.getChild("root"); + NullableMapVector mapVector = (NullableMapVector) parent.getChild("root"); TransferPair tp = mapVector.getTransferPair(allocator); tp.splitAndTransfer(0, 1); MapVector toMapVector = (MapVector) tp.getTo(); @@ -849,7 +849,7 @@ public void testSingleMapWriter1() { /* initialize a SingleMapWriter with empty MapVector and then lazily * create all vectors with expected initialCapacity. */ - NonNullableMapVector parent = NonNullableMapVector.empty("parent", allocator); + MapVector parent = MapVector.empty("parent", allocator); SingleMapWriter singleMapWriter = new SingleMapWriter(parent); int initialCapacity = 1024; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 1a7f71c72b4..9eb55c9bbda 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -36,7 +36,7 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.impl.ComplexWriterImpl; import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -94,7 +94,7 @@ public void tearDown() { DateTimeZone.setDefault(defaultTimezone); } - protected void writeData(int count, MapVector parent) { + protected void writeData(int count, NullableMapVector parent) { ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); IntWriter intWriter = rootWriter.integer("int"); @@ -119,7 +119,7 @@ protected void validateContent(int count, VectorSchemaRoot root) { } } - protected void writeComplexData(int count, MapVector parent) { + protected void writeComplexData(int count, NullableMapVector parent) { ArrowBuf varchar = allocator.buffer(3); varchar.readerIndex(0); varchar.setByte(0, 'a'); @@ -189,7 +189,7 @@ private LocalDateTime makeDateTimeFromCount(int i) { return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i); } - protected void writeDateTimeData(int count, MapVector parent) { + protected void writeDateTimeData(int count, NullableMapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -494,7 +494,7 @@ public void validateUnionData(int count, VectorSchemaRoot root) { } } - public void writeUnionData(int count, MapVector parent) { + public void writeUnionData(int count, NullableMapVector parent) { ArrowBuf varchar = allocator.buffer(3); varchar.readerIndex(0); varchar.setByte(0, 'a'); @@ -537,7 +537,7 @@ public void writeUnionData(int count, MapVector parent) { varchar.release(); } - protected void writeVarBinaryData(int count, MapVector parent) { + protected void writeVarBinaryData(int count, NullableMapVector parent) { Assert.assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java index 0cfc9ba919d..055c34e7010 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java @@ -42,8 +42,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; import org.apache.arrow.vector.complex.FixedSizeListVector; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.ipc.message.ArrowBlock; import org.apache.arrow.vector.ipc.message.ArrowBuffer; @@ -70,7 +69,7 @@ public void testWrite() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, new ByteArrayOutputStream()); } @@ -82,7 +81,7 @@ public void testWriteComplex() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); FieldVector root = parent.getChild("root"); validateComplexContent(count, new VectorSchemaRoot(root)); @@ -98,7 +97,7 @@ public void testWriteRead() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, stream); } @@ -154,7 +153,7 @@ public void testWriteReadComplex() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); write(parent.getChild("root"), file, stream); } @@ -195,7 +194,7 @@ public void testWriteReadMultipleRBs() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator); + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator); FileOutputStream fileOutputStream = new FileOutputStream(file)) { writeData(counts[0], parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); @@ -268,7 +267,7 @@ public void testWriteReadUnion() throws IOException { // write try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); validateUnionData(count, new VectorSchemaRoot(parent.getChild("root"))); write(parent.getChild("root"), file, stream); @@ -382,7 +381,7 @@ public void testWriteReadMetadata() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector vector = (MapVector) field.createVector(originalVectorAllocator)) { + NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { vector.allocateNewSafe(); vector.setValueCount(0); @@ -558,7 +557,7 @@ public void testWriteReadFixedSizeList() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class); Float4Vector floats = (Float4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); IntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), IntVector.class); @@ -618,7 +617,7 @@ public void testWriteReadVarBin() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index 4f9093b8c02..6a23a8438de 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -24,8 +24,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.complex.MapVector; -import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; import org.apache.arrow.vector.types.pojo.Schema; @@ -45,7 +44,7 @@ public void testWriteRead() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeData(count, parent); writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null); } @@ -73,7 +72,7 @@ public void testWriteReadComplexJSON() throws IOException { // write try ( BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null); } @@ -100,7 +99,7 @@ public void testWriteComplexJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateComplexContent(root.getRowCount(), root); @@ -122,7 +121,7 @@ public void testWriteReadUnionJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); printVectors(parent.getChildrenFromFields()); @@ -154,7 +153,7 @@ public void testWriteReadDateTimeJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeDateTimeData(count, parent); @@ -319,7 +318,7 @@ public void testWriteReadVarBinJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = MapVector.empty("parent", vectorAllocator)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeVarBinaryData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateVarBinary(count, root);