ARROW-482 [Java] Exposing custom field metadata

java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       write(parent.getChild("root"), testInFile);
     }

java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
     BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
 
     Field field = new Field(
-        "testField", true,
-        new ArrowType.Int(8, true),
+        "testField",
+        new FieldType(true, new ArrowType.Int(8, true), null, null),
         Collections.<Field>emptyList());
     NullableTinyIntVector vector =
         new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
         NullableIntVector writeVector =
             new NullableIntVector(
                 "varchar",
-                new FieldType(true, MinorType.INT.getType(), writeEncoding),
+                new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
                 allocator);
         NullableVarCharVector writeDictionaryVector =
             new NullableVarCharVector(
@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
          NullableVarCharVector writeDictionaryVector =
              new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
-         ListVector writeVector = new ListVector("list", allocator, null, null)) {
+         ListVector writeVector = ListVector.empty("list", allocator)) {
 
       // data being written:
       // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
       writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
       writeDictionaryVector.getMutator().setValueCount(2);
 
-      writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
+      writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
       writeVector.allocateNew();
       UnionListWriter listWriter = new UnionListWriter(writeVector);
       listWriter.startList();

java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
       MapWriter rootWriter = writer.rootAsMap();
       Float8Writer floatWriter = rootWriter.float8("float");
@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
             Integer.MAX_VALUE);
-        MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+        MapVector parent = MapVector.empty("parent", vectorAllocator)) {
       writeData(count, parent);
       ComplexWriter writer = new ComplexWriterImpl("root", parent);
       MapWriter rootWriter = writer.rootAsMap();

java/vector/src/main/codegen/templates/NullReader.java

@@ -55,10 +55,9 @@ public MinorType getMinorType() {
     return type;
   }
 
-
   @Override
   public Field getField() {
-    return new Field("", true, new Null(), null);
+    return new Field("", FieldType.nullable(new Null()), null);
   }
 
   public void copyAsValue(MapWriter writer) {}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private final int scale;
 
   public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
-    this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
+    this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
   }
   <#else>
   public ${className}(String name, BufferAllocator allocator) {
-    this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
+    this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
   }
   </#if>
 

java/vector/src/main/codegen/templates/UnionVector.java

@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
   public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
     this.name = name;
     this.allocator = allocator;
-    this.internalMap = new MapVector("internal", allocator, callBack);
+    this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
     this.typeVector = new UInt1Vector("types", allocator);
     this.callBack = callBack;
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
   }
 
   private FieldType fieldType(MinorType type) {
-    return new FieldType(true, type.getType(), null);
+    return FieldType.nullable(type.getType());
   }
 
   private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
@@ -250,7 +250,7 @@ public Field getField() {
       typeIds[childFields.size()] = v.getMinorType().ordinal();
       childFields.add(v.getField());
     }
-    return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
+    return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
   }
 
   @Override

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -29,6 +29,7 @@
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.TransferPair;
 
@@ -98,7 +99,7 @@ public void clear() { }
 
   @Override
   public Field getField() {
-    return new Field(name, true, new Null(), null);
+    return new Field(name, FieldType.nullable(new Null()), null);
   }
 
   @Override

java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

@@ -26,6 +26,11 @@
 import java.util.List;
 import java.util.Objects;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -48,44 +53,52 @@
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
 
+  public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
+    FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
+    return new FixedSizeListVector(name, allocator, fieldType, null);
+  }
+
   private FieldVector vector;
   private final BitVector bits;
   private final int listSize;
-  private final DictionaryEncoding dictionary;
+  private final FieldType fieldType;
   private final List<BufferBacked> innerVectors;
 
   private UnionFixedSizeListReader reader;
 
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public FixedSizeListVector(String name,
                              BufferAllocator allocator,
                              int listSize,
                              DictionaryEncoding dictionary,
                              CallBack schemaChangeCallback) {
+    this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
+  }
+
+  public FixedSizeListVector(String name,
+                             BufferAllocator allocator,
+                             FieldType fieldType,
+                             CallBack schemaChangeCallback) {
     super(name, allocator);
-    Preconditions.checkArgument(listSize > 0, "list size must be positive");
     this.bits = new BitVector("$bits$", allocator);
     this.vector = ZeroVector.INSTANCE;
-    this.listSize = listSize;
-    this.dictionary = dictionary;
+    this.fieldType = fieldType;
+    this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
+    Preconditions.checkArgument(listSize > 0, "list size must be positive");
     this.innerVectors = Collections.singletonList((BufferBacked) bits);
     this.reader = new UnionFixedSizeListReader(this);
   }
 
   @Override
   public Field getField() {
     List<Field> children = ImmutableList.of(getDataVector().getField());
-    return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
+    return new Field(name, fieldType, children);
   }
 
   @Override
@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
       throw new IllegalArgumentException("Lists have only one child. Found: " + children);
     }
     Field field = children.get(0);
-    FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
-    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
+    AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
     if (!addOrGetVector.isCreated()) {
       throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
     }
@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
     TransferPair pairs[] = new TransferPair[2];
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
-      this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
+      this(new FixedSizeListVector(name, allocator, fieldType, callBack));
     }
 
     public TransferImpl(FixedSizeListVector to) {
       this.to = to;
-      Field field = vector.getField();
-      FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
-      to.addOrGetVector(type);
+      to.addOrGetVector(vector.getField().getFieldType());
       pairs[0] = bits.makeTransferPair(to.bits);
       pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
     }

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -18,12 +18,17 @@
  ******************************************************************************/
 package org.apache.arrow.vector.complex;
 
+import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.AddOrGetResult;
@@ -41,40 +46,48 @@
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
 
+  public static ListVector empty(String name, BufferAllocator allocator) {
+    return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
+  }
+
   final UInt4Vector offsets;
   final BitVector bits;
   private final List<BufferBacked> innerVectors;
   private Mutator mutator = new Mutator();
   private Accessor accessor = new Accessor();
   private UnionListReader reader;
   private CallBack callBack;
-  private final DictionaryEncoding dictionary;
+  private final FieldType fieldType;
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
-    this(name, allocator, null, callBack);
+    this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
   }
 
+  // deprecated, use FieldType or static constructor instead
+  @Deprecated
   public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+    this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
+  }
+
+  public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
     super(name, allocator, callBack);
     this.bits = new BitVector("$bits$", allocator);
     this.offsets = getOffsetVector();
     this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
     this.reader = new UnionListReader(this);
-    this.dictionary = dictionary;
+    this.fieldType = checkNotNull(fieldType);
     this.callBack = callBack;
   }
 
@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
     TransferPair pairs[] = new TransferPair[3];
 
     public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
-      this(new ListVector(name, allocator, dictionary, callBack));
+      this(new ListVector(name, allocator, fieldType, callBack));
     }
 
     public TransferImpl(ListVector to) {
@@ -266,8 +279,7 @@ public int getBufferSize() {
 
   @Override
   public Field getField() {
-    return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
-            ImmutableList.of(getDataVector().getField()));
+    return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
   }
 
   @Override