diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index 34c93ed232c..6d9a6c1323e 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), testInFile); } diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 7cca33955d9..d7f6388721e 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException { BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE); Field field = new Field( - "testField", true, - new ArrowType.Int(8, true), + "testField", + new FieldType(true, new ArrowType.Int(8, true), null, null), Collections.emptyList()); NullableTinyIntVector vector = new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc); @@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException { NullableIntVector writeVector = new NullableIntVector( "varchar", - new FieldType(true, MinorType.INT.getType(), writeEncoding), + new FieldType(true, MinorType.INT.getType(), writeEncoding, null), allocator); NullableVarCharVector writeDictionaryVector = new NullableVarCharVector( @@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException { try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator); - ListVector writeVector = new ListVector("list", allocator, null, null)) { + ListVector writeVector = ListVector.empty("list", allocator)) { // data being written: // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]] @@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException { writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8)); writeDictionaryVector.getMutator().setValueCount(2); - writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding)); + writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null)); writeVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(writeVector); listWriter.startList(); diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java index 7d9a41985bb..4b840df670f 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java @@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double.. try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); Float8Writer floatWriter = rootWriter.float8("float"); @@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeData(count, parent); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java index 7c75b3ae1df..284a8d38f23 100644 --- a/java/vector/src/main/codegen/templates/NullReader.java +++ b/java/vector/src/main/codegen/templates/NullReader.java @@ -55,10 +55,9 @@ public MinorType getMinorType() { return type; } - @Override public Field getField() { - return new Field("", true, new Null(), null); + return new Field("", FieldType.nullable(new Null()), null); } public void copyAsValue(MapWriter writer) {} diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index 76d2bad36bc..ed2418e4a2a 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type private final int scale; public ${className}(String name, BufferAllocator allocator, int precision, int scale) { - this(name, new FieldType(true, new Decimal(precision, scale), null), allocator); + this(name, FieldType.nullable(new Decimal(precision, scale)), allocator); } <#else> public ${className}(String name, BufferAllocator allocator) { - this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator); + this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator); } diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 9d5dee5d237..ec42b12e809 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -77,7 +77,7 @@ public class UnionVector implements FieldVector { public UnionVector(String name, BufferAllocator allocator, CallBack callBack) { this.name = name; this.allocator = allocator; - this.internalMap = new MapVector("internal", allocator, callBack); + this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); this.typeVector = new UInt1Vector("types", allocator); this.callBack = callBack; this.innerVectors = Collections.unmodifiableList(Arrays.asList(typeVector)); @@ -125,7 +125,7 @@ private String fieldName(MinorType type) { } private FieldType fieldType(MinorType type) { - return new FieldType(true, type.getType(), null); + return FieldType.nullable(type.getType()); } private T addOrGet(MinorType minorType, Class c) { @@ -250,7 +250,7 @@ public Field getField() { typeIds[childFields.size()] = v.getMinorType().ordinal(); childFields.add(v.getField()); } - return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields); + return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index e48788c6ae7..01e22f25743 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -29,6 +29,7 @@ import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType.Null; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.TransferPair; @@ -98,7 +99,7 @@ public void clear() { } @Override public Field getField() { - return new Field(name, true, new Null(), null); + return new Field(name, FieldType.nullable(new Null()), null); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 0dceeed50d4..ee22fbec035 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -26,6 +26,11 @@ import java.util.List; import java.util.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.AddOrGetResult; @@ -48,18 +53,17 @@ import org.apache.arrow.vector.util.SchemaChangeRuntimeException; import org.apache.arrow.vector.util.TransferPair; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector { + public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size)); + return new FixedSizeListVector(name, allocator, fieldType, null); + } + private FieldVector vector; private final BitVector bits; private final int listSize; - private final DictionaryEncoding dictionary; + private final FieldType fieldType; private final List innerVectors; private UnionFixedSizeListReader reader; @@ -67,17 +71,26 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector, private Mutator mutator = new Mutator(); private Accessor accessor = new Accessor(); + // deprecated, use FieldType or static constructor instead + @Deprecated public FixedSizeListVector(String name, BufferAllocator allocator, int listSize, DictionaryEncoding dictionary, CallBack schemaChangeCallback) { + this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback); + } + + public FixedSizeListVector(String name, + BufferAllocator allocator, + FieldType fieldType, + CallBack schemaChangeCallback) { super(name, allocator); - Preconditions.checkArgument(listSize > 0, "list size must be positive"); this.bits = new BitVector("$bits$", allocator); this.vector = ZeroVector.INSTANCE; - this.listSize = listSize; - this.dictionary = dictionary; + this.fieldType = fieldType; + this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize(); + Preconditions.checkArgument(listSize > 0, "list size must be positive"); this.innerVectors = Collections.singletonList((BufferBacked) bits); this.reader = new UnionFixedSizeListReader(this); } @@ -85,7 +98,7 @@ public FixedSizeListVector(String name, @Override public Field getField() { List children = ImmutableList.of(getDataVector().getField()); - return new Field(name, true, new ArrowType.FixedSizeList(listSize), children); + return new Field(name, fieldType, children); } @Override @@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List children) { throw new IllegalArgumentException("Lists have only one child. Found: " + children); } Field field = children.get(0); - FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary()); - AddOrGetResult addOrGetVector = addOrGetVector(type); + AddOrGetResult addOrGetVector = addOrGetVector(field.getFieldType()); if (!addOrGetVector.isCreated()) { throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector()); } @@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair { TransferPair pairs[] = new TransferPair[2]; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack)); + this(new FixedSizeListVector(name, allocator, fieldType, callBack)); } public TransferImpl(FixedSizeListVector to) { this.to = to; - Field field = vector.getField(); - FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary()); - to.addOrGetVector(type); + to.addOrGetVector(vector.getField().getFieldType()); pairs[0] = bits.makeTransferPair(to.bits); pairs[1] = getDataVector().makeTransferPair(to.getDataVector()); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 9392afbccda..c2f55767ba6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -18,12 +18,17 @@ ******************************************************************************/ package org.apache.arrow.vector.complex; +import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.singletonList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.vector.AddOrGetResult; @@ -41,6 +46,7 @@ import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.schema.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -48,13 +54,12 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.TransferPair; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector { + public static ListVector empty(String name, BufferAllocator allocator) { + return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null); + } + final UInt4Vector offsets; final BitVector bits; private final List innerVectors; @@ -62,19 +67,27 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector, private Accessor accessor = new Accessor(); private UnionListReader reader; private CallBack callBack; - private final DictionaryEncoding dictionary; + private final FieldType fieldType; + // deprecated, use FieldType or static constructor instead + @Deprecated public ListVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, null, callBack); + this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack); } + // deprecated, use FieldType or static constructor instead + @Deprecated public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { + this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack); + } + + public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); this.bits = new BitVector("$bits$", allocator); this.offsets = getOffsetVector(); this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits, offsets)); this.reader = new UnionListReader(this); - this.dictionary = dictionary; + this.fieldType = checkNotNull(fieldType); this.callBack = callBack; } @@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair { TransferPair pairs[] = new TransferPair[3]; public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) { - this(new ListVector(name, allocator, dictionary, callBack)); + this(new ListVector(name, allocator, fieldType, callBack)); } public TransferImpl(ListVector to) { @@ -266,8 +279,7 @@ public int getBufferSize() { @Override public Field getField() { - return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(), - ImmutableList.of(getDataVector().getField())); + return new Field(name, fieldType, ImmutableList.of(getDataVector().getField())); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java index 997a6a38a08..bdd30f88f2c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java @@ -17,6 +17,8 @@ */ package org.apache.arrow.vector.complex; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -25,6 +27,10 @@ import javax.annotation.Nullable; +import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; +import com.google.common.primitives.Ints; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.FieldVector; @@ -33,26 +39,36 @@ import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.types.Types.MinorType; -import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.TransferPair; -import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; -import com.google.common.primitives.Ints; - public class MapVector extends AbstractMapVector { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); + public static MapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); + return new MapVector(name, allocator, fieldType, null); + } + private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this); private final Accessor accessor = new Accessor(); private final Mutator mutator = new Mutator(); + protected final FieldType fieldType; public int valueCount; + // deprecated, use FieldType or static constructor instead + @Deprecated public MapVector(String name, BufferAllocator allocator, CallBack callBack) { + this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack); + } + + public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { super(name, allocator, callBack); + this.fieldType = checkNotNull(fieldType); } @Override @@ -119,7 +135,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new MapTransferPair(this, new MapVector(name, allocator, callBack), false); + return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false); } @Override @@ -129,7 +145,7 @@ public TransferPair makeTransferPair(ValueVector to) { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new MapTransferPair(this, new MapVector(ref, allocator, callBack), false); + return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false); } protected static class MapTransferPair implements TransferPair{ @@ -212,8 +228,8 @@ public int getValueCapacity() { @Override public int compare(@Nullable ValueVector left, @Nullable ValueVector right) { return Ints.compare( - Preconditions.checkNotNull(left).getValueCapacity(), - Preconditions.checkNotNull(right).getValueCapacity() + checkNotNull(left).getValueCapacity(), + checkNotNull(right).getValueCapacity() ); } }; @@ -294,7 +310,7 @@ public Field getField() { for (ValueVector child : getChildren()) { children.add(child.getField()); } - return new Field(name, false, Struct.INSTANCE, children); + return new Field(name, fieldType, children); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java index 6456efba0dc..f1f86404d20 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java @@ -23,6 +23,9 @@ import java.util.Collections; import java.util.List; +import com.google.common.collect.ObjectArrays; + +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.BaseDataValueVector; import org.apache.arrow.vector.BitVector; @@ -32,39 +35,48 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl; import org.apache.arrow.vector.complex.impl.NullableMapWriter; -import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; import org.apache.arrow.vector.schema.ArrowFieldNode; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.TransferPair; -import com.google.common.collect.ObjectArrays; - -import io.netty.buffer.ArrowBuf; - public class NullableMapVector extends MapVector implements FieldVector { + public static NullableMapVector empty(String name, BufferAllocator allocator) { + FieldType fieldType = FieldType.nullable(Struct.INSTANCE); + return new NullableMapVector(name, allocator, fieldType, null); + } + private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this); private final NullableMapWriter writer = new NullableMapWriter(this); protected final BitVector bits; private final List innerVectors; - private final DictionaryEncoding dictionary; private final Accessor accessor; private final Mutator mutator; + // deprecated, use FieldType or static constructor instead + @Deprecated public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) { - this(name, allocator, null, callBack); + this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack); } + // deprecated, use FieldType or static constructor instead + @Deprecated public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) { - super(name, checkNotNull(allocator), callBack); + this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack); + } + + public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) { + super(name, checkNotNull(allocator), fieldType, callBack); this.bits = new BitVector("$bits$", allocator); this.innerVectors = Collections.unmodifiableList(Arrays.asList(bits)); - this.dictionary = dictionary; this.accessor = new Accessor(); this.mutator = new Mutator(); } @@ -96,7 +108,7 @@ public NullableMapWriter getWriter() { @Override public TransferPair getTransferPair(BufferAllocator allocator) { - return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, null), false); + return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false); } @Override @@ -106,12 +118,12 @@ public TransferPair makeTransferPair(ValueVector to) { @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, null), false); + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false); } @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) { - return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, callBack), false); + return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false); } protected class NullableMapTransferPair extends MapTransferPair { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java index 0666bc4137a..0f49ce61f1c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java @@ -29,6 +29,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; public class DictionaryEncoder { @@ -53,8 +54,9 @@ public static ValueVector encode(ValueVector vector, Dictionary dictionary) { } Field valueField = vector.getField(); - Field indexField = new Field(valueField.getName(), valueField.isNullable(), - dictionary.getEncoding().getIndexType(), dictionary.getEncoding(), null); + FieldType indexFieldType = new FieldType(valueField.isNullable(), dictionary.getEncoding().getIndexType(), + dictionary.getEncoding(), valueField.getMetadata()); + Field indexField = new Field(valueField.getName(), indexFieldType, null); // vector to hold our indices (dictionary encoded values) FieldVector indices = indexField.createVector(vector.getAllocator()); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java index 1d33913f71a..f00088577ff 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java @@ -40,6 +40,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; public abstract class ArrowReader implements DictionaryProvider, AutoCloseable { @@ -192,13 +193,13 @@ private Field toMemoryFormat(Field field, Map dictionaries) { // get existing or create dictionary vector if (!dictionaries.containsKey(encoding.getId())) { // create a new dictionary vector for the values - Field dictionaryField = new Field(field.getName(), field.isNullable(), field.getType(), null, children); + Field dictionaryField = new Field(field.getName(), new FieldType(field.isNullable(), field.getType(), null, null), children); FieldVector dictionaryVector = dictionaryField.createVector(allocator); dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding)); } } - return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren); + return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren); } private void load(ArrowDictionaryBatch dictionaryBatch) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java index 1716287f722..35b44fb682f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java @@ -36,6 +36,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,7 +118,7 @@ private Field toMessageFormat(Field field, DictionaryProvider provider, Map children, - @JsonProperty("typeLayout") TypeLayout typeLayout) { - this(name, new FieldType(nullable, type, dictionary), children, typeLayout); + @JsonProperty("typeLayout") TypeLayout typeLayout, + @JsonProperty("metadata") Map metadata) { + this(name, new FieldType(nullable, type, dictionary, metadata), children, typeLayout); } private Field(String name, FieldType fieldType, List children, TypeLayout typeLayout) { - super(); this.name = name; this.fieldType = checkNotNull(fieldType); - this.children = children == null ? ImmutableList.of() : children; + this.children = children == null ? ImmutableList.of() : ImmutableList.copyOf(children); this.typeLayout = checkNotNull(typeLayout); } - public Field(String name, FieldType fieldType, List children) { - this(name, fieldType, children, TypeLayout.getTypeLayout(fieldType.getType())); - } - + // deprecated, use FieldType or static constructor instead + @Deprecated public Field(String name, boolean nullable, ArrowType type, List children) { - this(name, nullable, type, null, children); + this(name, new FieldType(nullable, type, null, null), children); } + // deprecated, use FieldType or static constructor instead + @Deprecated public Field(String name, boolean nullable, ArrowType type, DictionaryEncoding dictionary, List children) { - this(name, new FieldType(nullable, type, dictionary), children); + this(name, new FieldType(nullable, type, dictionary, null), children); + } + + public Field(String name, FieldType fieldType, List children) { + this(name, fieldType, children, fieldType == null ? null : TypeLayout.getTypeLayout(fieldType.getType())); } public FieldVector createVector(BufferAllocator allocator) { @@ -114,7 +123,14 @@ public static Field convertField(org.apache.arrow.flatbuf.Field field) { childrenBuilder.add(convertField(field.children(i))); } List children = childrenBuilder.build(); - return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build())); + ImmutableMap.Builder metadataBuilder = ImmutableMap.builder(); + for (int i = 0; i < field.customMetadataLength(); i++) { + KeyValue kv = field.customMetadata(i); + String key = kv.key(), value = kv.value(); + metadataBuilder.put(key == null ? "" : key, value == null ? "" : value); + } + Map metadata = metadataBuilder.build(); + return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata); } public void validate() { @@ -147,7 +163,19 @@ public int getField(FlatBufferBuilder builder) { VectorLayout vectorLayout = typeLayout.getVectors().get(i); buffersData[i] = vectorLayout.writeTo(builder); } - int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData); + int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData); + int[] metadataOffsets = new int[getMetadata().size()]; + Iterator> metadataIterator = getMetadata().entrySet().iterator(); + for (int i = 0; i < metadataOffsets.length; i ++) { + Entry kv = metadataIterator.next(); + int keyOffset = builder.createString(kv.getKey()); + int valueOffset = builder.createString(kv.getValue()); + KeyValue.startKeyValue(builder); + KeyValue.addKey(builder, keyOffset); + KeyValue.addValue(builder, valueOffset); + metadataOffsets[i] = KeyValue.endKeyValue(builder); + } + int metadataOffset = org.apache.arrow.flatbuf.Field.createCustomMetadataVector(builder, metadataOffsets); org.apache.arrow.flatbuf.Field.startField(builder); if (name != null) { org.apache.arrow.flatbuf.Field.addName(builder, nameOffset); @@ -157,6 +185,7 @@ public int getField(FlatBufferBuilder builder) { org.apache.arrow.flatbuf.Field.addType(builder, typeOffset); org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset); org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset); + org.apache.arrow.flatbuf.Field.addCustomMetadata(builder, metadataOffset); if (dictionary != null) { org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset); } @@ -193,9 +222,14 @@ public TypeLayout getTypeLayout() { return typeLayout; } + @JsonInclude(Include.NON_EMPTY) + public Map getMetadata() { + return fieldType.getMetadata(); + } + @Override public int hashCode() { - return Objects.hash(name, isNullable(), getType(), getDictionary(), children); + return Objects.hash(name, isNullable(), getType(), getDictionary(), getMetadata(), children); } @Override @@ -205,10 +239,11 @@ public boolean equals(Object obj) { } Field that = (Field) obj; return Objects.equals(this.name, that.name) && - Objects.equals(this.isNullable(), that.isNullable()) && - Objects.equals(this.getType(), that.getType()) && - Objects.equals(this.getDictionary(), that.getDictionary()) && - Objects.equals(this.children, that.children); + Objects.equals(this.isNullable(), that.isNullable()) && + Objects.equals(this.getType(), that.getType()) && + Objects.equals(this.getDictionary(), that.getDictionary()) && + Objects.equals(this.getMetadata(), that.getMetadata()) && + Objects.equals(this.children, that.children); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java index fe99e631360..c8fc689cd2c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java @@ -19,6 +19,10 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.types.Types; @@ -28,18 +32,24 @@ public class FieldType { public static FieldType nullable(ArrowType type) { - return new FieldType(true, type, null); + return new FieldType(true, type, null, null); } private final boolean nullable; private final ArrowType type; private final DictionaryEncoding dictionary; + private final Map metadata; public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary) { + this(nullable, type, dictionary, null); + } + + public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary, Map metadata) { super(); this.nullable = nullable; this.type = checkNotNull(type); this.dictionary = dictionary; + this.metadata = metadata == null ? ImmutableMap.of() : ImmutableMap.copyOf(metadata); } public boolean isNullable() { @@ -51,6 +61,9 @@ public ArrowType getType() { public DictionaryEncoding getDictionary() { return dictionary; } + public Map getMetadata() { + return metadata; + } public FieldVector createNewSingleVector(String name, BufferAllocator allocator, CallBack schemaCallBack) { MinorType minorType = Types.getMinorTypeForArrowType(type); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index cfb7b3d2a26..f35ab54eaa8 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -49,7 +49,7 @@ public void terminate() throws Exception { @Test public void testIntType() { - try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); NullableIntVector.Mutator mutator = nested.getMutator(); vector.allocateNew(); @@ -77,7 +77,7 @@ public void testIntType() { @Test public void testFloatTypeNullable() { - try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); NullableFloat4Vector.Mutator mutator = nested.getMutator(); vector.allocateNew(); @@ -112,7 +112,7 @@ public void testFloatTypeNullable() { @Test public void testNestedInList() { - try (ListVector vector = new ListVector("list", allocator, null, null)) { + try (ListVector vector = ListVector.empty("list", allocator)) { ListVector.Mutator mutator = vector.getMutator(); FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector(); FixedSizeListVector.Mutator tupleMutator = tuples.getMutator(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 18d93b6401e..11be3298f75 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -42,8 +42,8 @@ public void terminate() throws Exception { @Test public void testCopyFrom() throws Exception { - try (ListVector inVector = new ListVector("input", allocator, null, null); - ListVector outVector = new ListVector("output", allocator, null, null)) { + try (ListVector inVector = ListVector.empty("input", allocator); + ListVector outVector = ListVector.empty("output", allocator)) { UnionListWriter writer = inVector.getWriter(); writer.allocate(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 40c7bc5ac9a..da9cb00361c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -123,7 +123,7 @@ public void testNullableType() { @Test public void testListType() { - try (final ListVector vector = new ListVector("", allocator, null)) { + try (final ListVector vector = ListVector.empty("", allocator)) { vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); vector.setInitialCapacity(512); @@ -146,7 +146,7 @@ public void testListType() { @Test public void testMapType() { - try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) { + try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) { vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class); vector.setInitialCapacity(512); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 372bcf0da6e..6fb559ca0d7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -42,6 +42,7 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.AfterClass; import org.junit.Assert; @@ -58,7 +59,7 @@ public void testUnloadLoad() throws IOException { try ( BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) { + MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) { // write some data ComplexWriter writer = new ComplexWriterImpl("root", parent); @@ -106,7 +107,7 @@ public void testUnloadLoadAddPadding() throws IOException { Schema schema; try ( BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) { + MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) { // write some data ComplexWriter writer = new ComplexWriterImpl("root", parent); @@ -182,8 +183,8 @@ public void testUnloadLoadAddPadding() throws IOException { @Test public void testLoadEmptyValidityBuffer() throws IOException { Schema schema = new Schema(asList( - new Field("intDefined", true, new ArrowType.Int(32, true), Collections.emptyList()), - new Field("intNull", true, new ArrowType.Int(32, true), Collections.emptyList()) + new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()), + new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.emptyList()) )); int count = 10; ArrowBuf validity = allocator.buffer(10).slice(0, 0); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 65b193c0aee..e826fa53b09 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -27,8 +27,10 @@ import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,7 +53,7 @@ public void terminate() throws Exception { @Test public void testPromoteToUnion() throws Exception { - try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null); + try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator); final NullableMapVector v = container.addOrGetMap("test"); final PromotableWriter writer = new PromotableWriter(v, container)) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index aba65dbf374..16139361756 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -17,16 +17,13 @@ */ package org.apache.arrow.vector.complex.writer; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.util.HashSet; import java.util.List; import java.util.Set; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.SchemaChangeCallBack; @@ -48,9 +45,11 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; +import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; @@ -60,8 +59,6 @@ import org.junit.Assert; import org.junit.Test; -import io.netty.buffer.ArrowBuf; - public class TestComplexWriter { private static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); @@ -101,7 +98,7 @@ public void transferPairSchemaChange() { } private MapVector populateMapVector(CallBack callBack) { - MapVector parent = new MapVector("parent", allocator, callBack); + MapVector parent = new MapVector("parent", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); IntWriter intWriter = rootWriter.integer("int"); @@ -118,7 +115,7 @@ private MapVector populateMapVector(CallBack callBack) { @Test public void nullableMap() { - try (MapVector mapVector = new MapVector("parent", allocator, null)) { + try (MapVector mapVector = MapVector.empty("parent", allocator)) { ComplexWriter writer = new ComplexWriterImpl("root", mapVector); MapWriter rootWriter = writer.rootAsMap(); for (int i = 0; i < COUNT; i++) { @@ -142,7 +139,7 @@ public void nullableMap() { */ @Test public void nullableMap2() { - try (MapVector mapVector = new MapVector("parent", allocator, null)) { + try (MapVector mapVector = MapVector.empty("parent", allocator)) { ComplexWriter writer = new ComplexWriterImpl("root", mapVector); MapWriter rootWriter = writer.rootAsMap(); MapWriter mapWriter = rootWriter.map("map"); @@ -181,7 +178,7 @@ private void checkNullableMap(MapVector mapVector) { @Test public void testList() { - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -210,7 +207,7 @@ public void testList() { @Test public void listScalarType() { - ListVector listVector = new ListVector("list", allocator, null, null); + ListVector listVector = ListVector.empty("list", allocator); listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); for (int i = 0; i < COUNT; i++) { @@ -233,7 +230,7 @@ public void listScalarType() { @Test public void listScalarTypeNullable() { - ListVector listVector = new ListVector("list", allocator, null, null); + ListVector listVector = ListVector.empty("list", allocator); listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); for (int i = 0; i < COUNT; i++) { @@ -262,7 +259,7 @@ public void listScalarTypeNullable() { @Test public void listMapType() { - ListVector listVector = new ListVector("list", allocator, null, null); + ListVector listVector = ListVector.empty("list", allocator); listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); MapWriter mapWriter = listWriter.map(); @@ -290,7 +287,7 @@ public void listMapType() { @Test public void listListType() { - try (ListVector listVector = new ListVector("list", allocator, null, null)) { + try (ListVector listVector = ListVector.empty("list", allocator)) { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); for (int i = 0; i < COUNT; i++) { @@ -315,7 +312,7 @@ public void listListType() { */ @Test public void listListType2() { - try (ListVector listVector = new ListVector("list", allocator, null, null)) { + try (ListVector listVector = ListVector.empty("list", allocator)) { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); ListWriter innerListWriter = listWriter.list(); @@ -353,7 +350,7 @@ private void checkListOfLists(final ListVector listVector) { @Test public void unionListListType() { - try (ListVector listVector = new ListVector("list", allocator, null, null)) { + try (ListVector listVector = ListVector.empty("list", allocator)) { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); for (int i = 0; i < COUNT; i++) { @@ -382,7 +379,7 @@ public void unionListListType() { */ @Test public void unionListListType2() { - try (ListVector listVector = new ListVector("list", allocator, null, null)) { + try (ListVector listVector = ListVector.empty("list", allocator)) { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); ListWriter innerListWriter = listWriter.list(); @@ -454,7 +451,7 @@ public void simpleUnion() { @Test public void promotableWriter() { - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); for (int i = 0; i < 100; i++) { @@ -503,7 +500,7 @@ public void promotableWriter() { */ @Test public void promotableWriterSchema() { - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); rootWriter.bigInt("a"); @@ -536,7 +533,7 @@ private Set getFieldNames(List fields) { @Test public void mapWriterMixedCaseFieldNames() { // test case-sensitive MapWriter - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true); MapWriter rootWriterCaseSensitive = writer.rootAsMap(); rootWriterCaseSensitive.bigInt("int_field"); @@ -607,7 +604,7 @@ public void timeStampWriters() throws Exception { final LocalDateTime expectedNanoDateTime = expectedMilliDateTime; // write - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); @@ -678,7 +675,7 @@ public void timeStampWriters() throws Exception { @Test public void complexCopierWithList() { - MapVector parent = new MapVector("parent", allocator, null); + MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); ListWriter listWriter = rootWriter.list("list"); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java index 3bed45361fc..9156110a863 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java @@ -27,10 +27,13 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.arrow.memory.BufferAllocator; @@ -55,7 +58,9 @@ import org.apache.arrow.vector.stream.ArrowStreamReader; import org.apache.arrow.vector.stream.ArrowStreamWriter; import org.apache.arrow.vector.stream.MessageSerializerTest; +import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; @@ -77,7 +82,7 @@ public void testWrite() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", vectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, new ByteArrayOutputStream()); } @@ -89,7 +94,7 @@ public void testWriteComplex() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); FieldVector root = parent.getChild("root"); validateComplexContent(count, new VectorSchemaRoot(root)); @@ -105,7 +110,7 @@ public void testWriteRead() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { writeData(count, parent); write(parent.getChild("root"), file, stream); } @@ -172,7 +177,7 @@ public void testWriteReadComplex() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); write(parent.getChild("root"), file, stream); } @@ -213,7 +218,7 @@ public void testWriteReadMultipleRBs() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorAllocator, null); + MapVector parent = MapVector.empty("parent", originalVectorAllocator); FileOutputStream fileOutputStream = new FileOutputStream(file)){ writeData(counts[0], parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); @@ -286,7 +291,7 @@ public void testWriteReadUnion() throws IOException { // write try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); validateUnionData(count, new VectorSchemaRoot(parent.getChild("root"))); write(parent.getChild("root"), file, stream); @@ -380,6 +385,77 @@ private void validateTinyData(VectorSchemaRoot root) { } } + @Test + public void testWriteReadFieldMetadata() throws IOException { + File file = new File("target/mytest_metadata.arrow"); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + List childFields = new ArrayList(); + childFields.add(new Field("varchar-child", new FieldType(true, ArrowType.Utf8.INSTANCE, null, metadata(1)), null)); + childFields.add(new Field("float-child", new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null, metadata(2)), null)); + childFields.add(new Field("int-child", new FieldType(false, new ArrowType.Int(32, true), null, metadata(3)), null)); + childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)), + ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16 ,true)), null)))); + Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields); + List fields = ImmutableList.of(field); + + // write + try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); + NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) { + vector.allocateNewSafe(); + vector.getMutator().setValueCount(0); + + List vectors = ImmutableList.of(vector); + VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0); + + try (FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel()); + ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) { + LOGGER.debug("writing schema: " + root.getSchema()); + fileWriter.start(); + streamWriter.start(); + fileWriter.writeBatch(); + streamWriter.writeBatch(); + fileWriter.end(); + streamWriter.end(); + } + } + + // read from file + try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); + FileInputStream fileInputStream = new FileInputStream(file); + ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) { + VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); + Schema schema = root.getSchema(); + LOGGER.debug("reading schema: " + schema); + Assert.assertEquals(fields, schema.getFields()); + Field top = schema.getFields().get(0); + Assert.assertEquals(metadata(0), top.getMetadata()); + for (int i = 0; i < 4; i ++) { + Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata()); + } + } + + // Read from stream + try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE); + ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray()); + ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) { + VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); + Schema schema = root.getSchema(); + LOGGER.debug("reading schema: " + schema); + Assert.assertEquals(fields, schema.getFields()); + Field top = schema.getFields().get(0); + Assert.assertEquals(metadata(0), top.getMetadata()); + for (int i = 0; i < 4; i ++) { + Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata()); + } + } + } + + private Map metadata(int i) { + return ImmutableMap.of("k_" + i, "v_" + i, "k2_" + i, "v2_" + i); + } + @Test public void testWriteReadDictionary() throws IOException { File file = new File("target/mytest_dict.arrow"); @@ -491,7 +567,7 @@ public void testWriteReadNestedDictionary() throws IOException { // write try (NullableVarCharVector dictionaryVector = newNullableVarCharVector("dictionary", allocator); - ListVector listVector = new ListVector("list", allocator, null, null)) { + ListVector listVector = ListVector.empty("list", allocator)) { Dictionary dictionary = new Dictionary(dictionaryVector, encoding); MapDictionaryProvider provider = new MapDictionaryProvider(); @@ -590,10 +666,10 @@ public void testWriteReadFixedSizeList() throws IOException { // write try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", originalVectorAllocator, null, null)) { - FixedSizeListVector tuples = parent.addOrGet("float-pairs", new FieldType(true, new FixedSizeList(2), null), FixedSizeListVector.class); - NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(new FieldType(true, MinorType.FLOAT4.getType(), null)).getVector(); - NullableIntVector ints = parent.addOrGet("ints", new FieldType(true, new Int(32, true), null), NullableIntVector.class); + NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) { + FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class); + NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector(); + NullableIntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), NullableIntVector.class); parent.allocateNew(); for (int i = 0; i < 10; i++) { @@ -641,6 +717,7 @@ public void testWriteReadFixedSizeList() throws IOException { } } + /** * Writes the contents of parents to file. If outStream is non-null, also writes it * to outStream in the streaming serialized format. diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java index 1e514585e50..3014e64b4ee 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java @@ -28,6 +28,7 @@ import org.apache.arrow.flatbuf.Footer; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Test; @@ -38,7 +39,7 @@ public class TestArrowFooter { @Test public void test() { Schema schema = new Schema(asList( - new Field("a", true, new ArrowType.Int(8, true), Collections.emptyList()) + new Field("a", FieldType.nullable(new ArrowType.Int(8, true)), Collections.emptyList()) )); ArrowFooter footer = new ArrowFooter(schema, Collections.emptyList(), Collections.emptyList()); ArrowFooter newFooter = roundTrip(footer); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java index d00cb0f8c00..55629d5107c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java @@ -41,6 +41,7 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.junit.Before; @@ -71,7 +72,7 @@ byte[] array(ArrowBuf buf) { @Test public void test() throws IOException { - Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.emptyList()))); + Schema schema = new Schema(asList(new Field("testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.emptyList()))); ArrowType type = schema.getFields().get(0).getType(); FieldVector vector = TestUtils.newVector(FieldVector.class, "testField", type, allocator); vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java index 6369c07c320..6c29cab1a07 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java @@ -43,7 +43,7 @@ public void testWriteReadComplexJSON() throws IOException { // write try ( BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", originalVectorAllocator, null)) { + MapVector parent = MapVector.empty("parent", originalVectorAllocator)) { writeComplexData(count, parent); writeJSON(file, new VectorSchemaRoot(parent.getChild("root"))); } @@ -70,7 +70,7 @@ public void testWriteComplexJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeComplexData(count, parent); VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateComplexContent(root.getRowCount(), root); @@ -92,7 +92,7 @@ public void testWriteReadUnionJSON() throws IOException { int count = COUNT; try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeUnionData(count, parent); @@ -127,7 +127,7 @@ public void testWriteReadDateTimeJSON() throws IOException { // write try ( BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) { + NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) { writeDateTimeData(count, parent); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index f9c8f726ab6..64f797030ff 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -32,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Test; @@ -45,25 +46,25 @@ public class TestConvert { @Test public void simple() { - Field initialField = new Field("a", true, new Int(32, true), null); + Field initialField = new Field("a", FieldType.nullable(new Int(32, true)), null); run(initialField); } @Test public void complex() { ImmutableList.Builder childrenBuilder = ImmutableList.builder(); - childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null)); - childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.of())); + childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null)); + childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.of())); - Field initialField = new Field("a", true, Struct.INSTANCE, childrenBuilder.build()); + Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build()); run(initialField); } @Test public void schema() { ImmutableList.Builder childrenBuilder = ImmutableList.builder(); - childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null)); - childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.of())); + childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null)); + childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.of())); Schema initialSchema = new Schema(childrenBuilder.build()); run(initialSchema); } @@ -71,18 +72,18 @@ public void schema() { @Test public void nestedSchema() { ImmutableList.Builder childrenBuilder = ImmutableList.builder(); - childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null)); - childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.of())); - childrenBuilder.add(new Field("child3", true, new Struct(), ImmutableList.of( - new Field("child3.1", true, Utf8.INSTANCE, null), - new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.of()) + childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null)); + childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.of())); + childrenBuilder.add(new Field("child3", FieldType.nullable(new Struct()), ImmutableList.of( + new Field("child3.1", FieldType.nullable(Utf8.INSTANCE), null), + new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.of()) ))); - childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.of( - new Field("child4.1", true, Utf8.INSTANCE, null) + childrenBuilder.add(new Field("child4", FieldType.nullable(new List()), ImmutableList.of( + new Field("child4.1", FieldType.nullable(Utf8.INSTANCE), null) ))); - childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.of( - new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null), - new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.of()) + childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.of( + new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null), + new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.of()) ))); Schema initialSchema = new Schema(childrenBuilder.build()); run(initialSchema); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java index 27879efeaf1..9678423c0fb 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.List; +import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.file.ArrowBlock; @@ -40,13 +41,12 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Test; import org.junit.Rule; +import org.junit.Test; import org.junit.rules.ExpectedException; -import io.netty.buffer.ArrowBuf; - public class MessageSerializerTest { public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) { @@ -79,7 +79,7 @@ public void testSchemaMessageSerialization() throws IOException { @Test public void testSchemaDictionaryMessageSerialization() throws IOException { DictionaryEncoding dictionary = new DictionaryEncoding(9L, false, new ArrowType.Int(8, true)); - Field field = new Field("test", true, ArrowType.Utf8.INSTANCE, dictionary, null); + Field field = new Field("test", new FieldType(true, ArrowType.Utf8.INSTANCE, dictionary, null), null); Schema schema = new Schema(Collections.singletonList(field)); ByteArrayOutputStream out = new ByteArrayOutputStream(); long size = MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), schema); @@ -130,7 +130,7 @@ public void testSerializeRecordBatch() throws IOException { public static Schema testSchema() { return new Schema(asList(new Field( - "testField", true, new ArrowType.Int(8, true), Collections.emptyList()))); + "testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.emptyList()))); } // Verifies batch contents matching test schema. diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java index 56fa73ecceb..84cc10787f7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java @@ -47,7 +47,7 @@ public class TestSchema { private static Field field(String name, boolean nullable, ArrowType type, Field... children) { - return new Field(name, nullable, type, asList(children)); + return new Field(name, new FieldType(nullable, type, null, null), asList(children)); } private static Field field(String name, ArrowType type, Field... children) {