diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java index dc708724043..211718e1ebd 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java @@ -45,6 +45,7 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -83,6 +84,11 @@ public ColumnBinder visit(ArrowType.List type) { return new ListBinder((ListVector) vector); } + @Override + public ColumnBinder visit(ArrowType.ListView type) { + return new ListBinder((ListViewVector) vector); + } + @Override public ColumnBinder visit(ArrowType.LargeList type) { throw new UnsupportedOperationException("No column binder implemented for type " + type); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/ListViewAvaticaParameterConverter.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/ListViewAvaticaParameterConverter.java new file mode 100644 index 00000000000..cb3cdc1543b --- /dev/null +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/converter/impl/ListViewAvaticaParameterConverter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.driver.jdbc.converter.impl; + +import java.util.List; + +import org.apache.arrow.driver.jdbc.utils.AvaticaParameterBinder; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.calcite.avatica.AvaticaParameter; +import org.apache.calcite.avatica.remote.TypedValue; + +/** + * AvaticaParameterConverter for List Arrow types. + */ +public class ListViewAvaticaParameterConverter extends BaseAvaticaParameterConverter { + + public ListViewAvaticaParameterConverter(ArrowType.ListView type) { + } + + //FIXME! Add unit test to validate this bindParameter + @Override + public boolean bindParameter(FieldVector vector, TypedValue typedValue, int index) { + final List values = (List) typedValue.value; + + if (vector instanceof ListViewVector) { + ListViewVector listViewVector = ((ListViewVector) vector); + FieldVector childVector = listViewVector.getDataVector(); + + int startPos = listViewVector.startNewValue(index); + for (int i = 0; i < values.size(); i++) { + Object val = values.get(i); + int childIndex = startPos + i; + if (val == null) { + if (childVector.getField().isNullable()) { + childVector.setNull(childIndex); + } else { + throw new UnsupportedOperationException("Can't set null on non-nullable child list"); + } + } else { + childVector.getField().getType().accept( + new AvaticaParameterBinder.BinderVisitor( + childVector, TypedValue.ofSerial(typedValue.componentType, val), childIndex)); + } + } + listViewVector.endValue(index, values.size()); + listViewVector.setValueCount(index + 1); + return true; + } + return false; + } + + @Override + public AvaticaParameter createParameter(Field field) { + return createParameter(field, false); + } +} diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java index b2bd8e745ec..2df71ee1d26 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/AvaticaParameterBinder.java @@ -34,6 +34,7 @@ import org.apache.arrow.driver.jdbc.converter.impl.LargeListAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.LargeUtf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.ListAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.ListViewAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.MapAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.NullAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.StructAvaticaParameterConverter; @@ -155,6 +156,11 @@ public Boolean visit(ArrowType.List type) { return new ListAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); } + @Override + public Boolean visit(ArrowType.ListView type) { + return new ListViewAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); + } + @Override public Boolean visit(ArrowType.LargeList type) { return new LargeListAvaticaParameterConverter(type).bindParameter(vector, typedValue, index); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index 843fe0cb89d..8baa50056d1 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -36,6 +36,7 @@ import org.apache.arrow.driver.jdbc.converter.impl.LargeListAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.LargeUtf8AvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.ListAvaticaParameterConverter; +import org.apache.arrow.driver.jdbc.converter.impl.ListViewAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.MapAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.NullAvaticaParameterConverter; import org.apache.arrow.driver.jdbc.converter.impl.StructAvaticaParameterConverter; @@ -170,6 +171,12 @@ public AvaticaParameter visit(ArrowType.List type) { } + @Override + public AvaticaParameter visit(ArrowType.ListView type) { + return new ListViewAvaticaParameterConverter(type).createParameter(field); + + } + @Override public AvaticaParameter visit(ArrowType.LargeList type) { return new LargeListAvaticaParameterConverter(type).createParameter(field); diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd index 3cf9a968791..72d666b5fe8 100644 --- a/java/vector/src/main/codegen/data/ArrowTypes.tdd +++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd @@ -119,6 +119,11 @@ name: "Duration", fields: [{name: "unit", type: short, valueType: TimeUnit}], complex: false + }, + { + name: "ListView", + fields: [], + complex: true } ] } diff --git a/java/vector/src/main/codegen/templates/UnionListViewWriter.java b/java/vector/src/main/codegen/templates/UnionListViewWriter.java new file mode 100644 index 00000000000..d73de4d2bc7 --- /dev/null +++ b/java/vector/src/main/codegen/templates/UnionListViewWriter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.impl.UnionListWriter; + +<@pp.dropOutputFile /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionListViewWriter.java" /> + + +<#include "/@includes/license.ftl" /> + +package org.apache.arrow.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> + +/* + * This class is generated using freemarker and the ${.template_name} template. + */ + +/** + *

Writer for ListViewVector. This extends UnionListWriter to simplify writing listview entries to a list + *

+ */ +@SuppressWarnings("unused") +public class UnionListViewWriter extends UnionListWriter { + public UnionListViewWriter(ListViewVector vector) { + super(vector); + } + + public void startList(int offset) { + ((ListViewVector) vector).startNewValue(idx(), offset); + writer.setPosition(((ListViewVector) vector).getOffsetBuffer().getInt((idx()) * 4)); + } + + public void endList(int size) { + ((ListViewVector) vector).endValue(idx(), size); + setPosition(idx() + 1); + } + + public void setValueCount(int count) { + ((ListViewVector) vector).setValueCount(count); + } +} diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index 822d4822987..a4c4b57edde 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -39,7 +39,7 @@ @SuppressWarnings("unused") public class UnionReader extends AbstractFieldReader { - private static final int NUM_SUPPORTED_TYPES = 46; + private static final int NUM_SUPPORTED_TYPES = 47; private BaseReader[] readers = new BaseReader[NUM_SUPPORTED_TYPES]; public UnionVector data; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java index ae465418cf2..3cd40318f56 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java @@ -108,6 +108,15 @@ public TypeLayout visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) return new TypeLayout(vectors); } + @Override + public TypeLayout visit(ArrowType.ListView type) { + List vectors = asList( + BufferLayout.validityVector(), + BufferLayout.offsetBuffer() + ); + return new TypeLayout(vectors); + } + @Override public TypeLayout visit(ArrowType.LargeList type) { List vectors = asList( @@ -304,6 +313,12 @@ public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { return 2; } + @Override + public Integer visit(ArrowType.ListView type) { + // validity buffer + offset buffer + sizes buffer + return 3; + } + @Override public Integer visit(ArrowType.LargeList type) { // validity buffer + offset buffer diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java new file mode 100644 index 00000000000..6d005529c15 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.complex; + +import java.util.List; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.BitVectorHelper; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.impl.UnionListViewWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.JsonStringArrayList; + +/** + * A list vector contains lists of a specific type of elements. Its structure contains 4 elements. + *
    + *
  1. A validity buffer.
    2. + *
  2. A child data vector that contains the elements of lists.
    3. + *
  3. An offset buffer, stores the starting index of each list.
    4. + *
  4. A size buffer, stored explicitly each list lengths.
    5. + *
+ * The latter two are managed by its superclass. + */ +public class ListViewVector extends ListVector { + protected ArrowBuf sizeBuffer; + protected long sizeAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; + + private int childArraySize = 0; + + /** + * Constructs a new instance. + * + * @param name The name of the instance. + * @param allocator The allocator to use for allocating/reallocating buffers. + * @param fieldType The type of this list. + * @param callBack A schema change callback. + */ + public ListViewVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, allocator, fieldType, callBack); + this.sizeBuffer = allocator.getEmpty(); + } + + /** + * Constructs a new instance. + * + * @param field The field materialized by this vector. + * @param allocator The allocator to use for allocating/reallocating buffers. + * @param callBack A schema change callback. + */ + public ListViewVector(Field field, BufferAllocator allocator, CallBack callBack) { + super(field, allocator, callBack); + this.sizeBuffer = allocator.getEmpty(); + } + + public static ListViewVector empty(String name, BufferAllocator allocator) { + return new ListViewVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); + } + + @Override + public void allocateNew() throws OutOfMemoryException { + if (!allocateNewSafe()) { + throw new OutOfMemoryException("Failure while allocating memory"); + } + } + + /** + * Allocate memory for the vector. We internally use a default value count + * of 4096 to allocate memory for at least these many elements in the + * vector. + * + * @return false if memory allocation fails, true otherwise. + */ + @Override + public boolean allocateNewSafe() { + boolean success = false; + try { + clear(); + success = super.allocateNewSafe(); + allocateSizeBuffer(sizeAllocationSizeInBytes); + } finally { + if (!success) { + clear(); + } + } + return success; + } + + @Override + public void clear() { + super.clear(); + sizeBuffer = releaseBuffer(sizeBuffer); + } + + @Override + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + if (valueCount > 0) { + while (valueCount > getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + } + vector.setValueCount(childArraySize); + } + + /** + * Start a new value in the list vector. + * + * @param index index of the value to start + * @param offset encode the start position of each slot in the child array + */ + public int startNewValue(int index, int offset) { + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + offsetBuffer.setInt(index * OFFSET_WIDTH, offset); + BitVectorHelper.setBit(validityBuffer, index); + return offsetBuffer.getInt(index * OFFSET_WIDTH); + } + + /** + * Update index values for offset and size buffer. + * + * @param index Current index position. + * @param size The number of items added. + */ + public void endValue(int index, int size) { + sizeBuffer.setInt(index * OFFSET_WIDTH, size); + int currentchildArraySize = size + offsetBuffer.getInt(index * OFFSET_WIDTH); + if (currentchildArraySize > childArraySize) { + childArraySize = currentchildArraySize; + } + } + + /** + * Get the element in the list vector at a particular index. + * @param index position of the element + * @return Object at given position + */ + @Override + public List getObject(int index) { + if (isSet(index) == 0) { + return null; + } + final List vals = new JsonStringArrayList<>(); + final ValueVector vv = getDataVector(); + int items = sizeBuffer.getInt(index * OFFSET_WIDTH); + int position = offsetBuffer.getInt(index * OFFSET_WIDTH); + for (int i = 0; i < items; i++) { + vals.add(vv.getObject(i + position)); + } + + return vals; + } + + /** + * Sets list at index to be null. + * @param index position in vector + */ + @Override + public void setNull(int index) { + while (index >= getValueCapacity()) { + reallocValidityAndOffsetBuffers(); + } + BitVectorHelper.unsetBit(validityBuffer, index); + } + + @Override + public UnionListViewWriter getWriter() { + return new UnionListViewWriter(this); + } + + protected void allocateSizeBuffer(final long size) { + final int curSize = (int) size; + sizeBuffer = allocator.buffer(curSize); + sizeBuffer.readerIndex(0); + sizeAllocationSizeInBytes = curSize; + sizeBuffer.setZero(0, sizeBuffer.capacity()); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 0b0e0d66a98..a702f7ea895 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -69,6 +69,7 @@ import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; @@ -131,6 +132,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.LargeBinary; import org.apache.arrow.vector.types.pojo.ArrowType.LargeUtf8; import org.apache.arrow.vector.types.pojo.ArrowType.List; +import org.apache.arrow.vector.types.pojo.ArrowType.ListView; import org.apache.arrow.vector.types.pojo.ArrowType.Map; import org.apache.arrow.vector.types.pojo.ArrowType.Null; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; @@ -658,6 +660,20 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { return new UnionListWriter((ListVector) vector); } }, + LISTVIEW(ListView.INSTANCE) { + @Override + public FieldVector getNewVector( + Field field, + BufferAllocator allocator, + CallBack schemaChangeCallback) { + return new ListViewVector(field.getName(), allocator, field.getFieldType(), schemaChangeCallback); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new UnionListWriter((ListVector) vector); + } + }, LARGELIST(ArrowType.LargeList.INSTANCE) { @Override public FieldVector getNewVector(Field field, BufferAllocator allocator, CallBack schemaChangeCallback) { @@ -861,6 +877,11 @@ public MinorType visit(List type) { return MinorType.LIST; } + @Override + public MinorType visit(ListView type) { + return MinorType.LISTVIEW; + } + @Override public MinorType visit(FixedSizeList type) { return MinorType.FIXED_SIZE_LIST; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java new file mode 100644 index 00000000000..b0a1f664689 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.impl.UnionListViewWriter; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestListViewVector { + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); + } + + @After + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testListView() throws Exception { + // values = [12, -7, 25, 0, -127, 127, 50] + // offsets = [0, 7, 3, 0] + // sizes = [3, 0, 4, 0] + // data to get thru listview: [[12,-7,25], null, [0,-127,127,50], []] + try (ListViewVector listViewVector = ListViewVector.empty("input", allocator); + IntVector inVector = + (IntVector) listViewVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())) + .getVector()) { + + listViewVector.allocateNew(); + inVector.allocateNew(); + + inVector.setSafe(0, 12); + inVector.setSafe(1, -7); + inVector.setSafe(2, 25); + inVector.setSafe(3, 0); + inVector.setSafe(4, -127); + inVector.setSafe(5, 127); + inVector.setSafe(6, 50); + + listViewVector.startNewValue(0, 0); + listViewVector.endValue(0, 3); + + listViewVector.setNull(1); + + listViewVector.startNewValue(2, 3); + listViewVector.endValue(2, 4); + + listViewVector.startNewValue(3, 0); + listViewVector.endValue(3, 0); + + listViewVector.setValueCount(4); + + assertEquals(inVector.toString(), "[12, -7, 25, 0, -127, 127, 50]"); + assertEquals(listViewVector.toString(), "[[12,-7,25], null, [0,-127,127,50], []]"); + } + } + + @Test + public void testListViewWithSharedValues() throws Exception { + // values = [0, -127, 127, 50, 12, -7, 25] + // offsets = [4, 7, 0, 0, 3] + // sizes = [3, 0, 4, 0, 2] + // data to get thru listview: [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] + // shared values = [50,] + try (ListViewVector listViewVector = ListViewVector.empty("input", allocator); + IntVector inVector = + (IntVector) listViewVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())) + .getVector()) { + + listViewVector.allocateNew(); + inVector.allocateNew(); + + inVector.setSafe(0, 0); + inVector.setSafe(1, -127); + inVector.setSafe(2, 127); + inVector.setSafe(3, 50); + inVector.setSafe(4, 12); + inVector.setSafe(5, -7); + inVector.setSafe(6, 25); + + listViewVector.startNewValue(0, 4); + listViewVector.endValue(0, 3); + + listViewVector.setNull(1); + + listViewVector.startNewValue(2, 0); + listViewVector.endValue(2, 4); + + listViewVector.startNewValue(3, 0); + listViewVector.endValue(3, 0); + + listViewVector.startNewValue(4, 3); + listViewVector.endValue(4, 2); + + listViewVector.setValueCount(5); + + assertEquals(inVector.toString(), "[0, -127, 127, 50, 12, -7, 25]"); + assertEquals(listViewVector.toString(), "[[12,-7,25], null, [0,-127,127,50], [], [50,12]]"); + } + } + + @Test + public void testPartialListViewWithSharedValues() throws Exception { + // values = [12, -7, 25, 17, -127, 200, 400, 600, 4, 6, 11, 0] + // offsets = [2, 5] + // sizes = [5, 3] + // data to get thru listview: [[25,17,-127,200], [200,400,600]] + // shared values = [200, 400, ] + try (ListViewVector listViewVector = ListViewVector.empty("input", allocator); + IntVector inVector = + (IntVector) listViewVector.addOrGetVector(FieldType.nullable(Types.MinorType.INT.getType())) + .getVector()) { + + listViewVector.allocateNew(); + inVector.allocateNew(); + + inVector.setSafe(0, 12); + inVector.setSafe(1, -7); + inVector.setSafe(2, 25); + inVector.setSafe(3, 17); + inVector.setSafe(4, -127); + inVector.setSafe(5, 200); + inVector.setSafe(6, 400); + inVector.setSafe(7, 600); + inVector.setSafe(8, 4); + inVector.setSafe(9, 6); + inVector.setSafe(10, 11); + inVector.setSafe(11, 0); + + listViewVector.startNewValue(0, 2); + listViewVector.endValue(0, 5); + + listViewVector.startNewValue(1, 5); + listViewVector.endValue(1, 3); + + listViewVector.setValueCount(2); + inVector.setValueCount(12); + + assertEquals(inVector.toString(), "[12, -7, 25, 17, -127, 200, 400, 600, 4, 6, 11, 0]"); + assertEquals(listViewVector.toString(), "[[25,17,-127,200,400], [200,400,600]]"); + } + } + + @Test + public void testListViewWriter() throws Exception { + // values = [12, -7, 25, 0, -127, 127, 50] + // offsets = [0, 7, 3, 0] + // sizes = [3, 0, 4, 0] + // data to get thru listview: [[12,-7,25], null, [0,-127,127,50], []] + try (ListViewVector inVector = ListViewVector.empty("input", allocator)) { + UnionListViewWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(0); + writer.bigInt().writeBigInt(12); + writer.bigInt().writeBigInt(-7); + writer.bigInt().writeBigInt(25); + writer.endList(3); + + + writer.setPosition(2); + writer.startList(3); + writer.bigInt().writeBigInt(0); + writer.bigInt().writeBigInt(-127); + writer.bigInt().writeBigInt(127); + writer.bigInt().writeBigInt(50); + writer.endList(4); + + writer.setPosition(3); + writer.startList(0); + writer.endList(0); + + writer.setValueCount(4); + + assertEquals(inVector.getDataVector().toString(), "[12, -7, 25, 0, -127, 127, 50]"); + assertEquals(inVector.toString(), "[[12,-7,25], null, [0,-127,127,50], []]"); + } + } + + @Test + public void testListViewWriterWithSharedValues() throws Exception { + // values = [0, -127, 127, 50, 12, -7, 25] + // offsets = [4, 7, 0, 0, 3] + // sizes = [3, 0, 4, 0, 2] + // data to get thru listview: [[12, -7, 25], null, [0, -127, 127, 50], [], [50, 12]] + // shared values = [50,] + try (ListViewVector inVector = ListViewVector.empty("input", allocator)) { + UnionListViewWriter writer = inVector.getWriter(); + writer.allocate(); + + writer.setPosition(0); + writer.startList(4); + writer.bigInt().writeBigInt(12); + writer.bigInt().writeBigInt(-7); + writer.bigInt().writeBigInt(25); + writer.endList(3); + + + writer.setPosition(2); + writer.startList(0); + writer.bigInt().writeBigInt(0); + writer.bigInt().writeBigInt(-127); + writer.bigInt().writeBigInt(127); + writer.bigInt().writeBigInt(50); + writer.endList(4); + + writer.setPosition(3); + writer.startList(0); + writer.endList(0); + + writer.setPosition(4); + writer.startList(3); + writer.bigInt().writeBigInt(50); + writer.bigInt().writeBigInt(12); + writer.endList(2); + + writer.setValueCount(5); + + assertEquals(inVector.getDataVector().toString(), "[0, -127, 127, 50, 12, -7, 25]"); + assertEquals(inVector.toString(), "[[12,-7,25], null, [0,-127,127,50], [], [50,12]]"); + } + } +}