ARROW-5902: [Java] Implement hash table and equals & hashCode API for dictionary encoding

java/vector/src/main/codegen/templates/UnionVector.java

@@ -17,6 +17,7 @@
 
 import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.ReferenceManager;
+import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.pojo.FieldType;
 
 <@pp.dropOutputFile />
@@ -491,32 +492,35 @@ public Iterator<ValueVector> iterator() {
     return vectors.iterator();
   }
 
-
-    public Object getObject(int index) {
+    private ValueVector getVector(int index) {
       int type = typeBuffer.getByte(index * TYPE_WIDTH);
       switch (MinorType.values()[type]) {
-      case NULL:
-        return null;
+        case NULL:
+          return null;
       <#list vv.types as type>
         <#list type.minor as minor>
           <#assign name = minor.class?cap_first />
           <#assign fields = minor.fields!type.fields />
           <#assign uncappedName = name?uncap_first/>
           <#if !minor.typeParams?? >
-      case ${name?upper_case}:
-          return get${name}Vector().getObject(index);
+        case ${name?upper_case}:
+        return get${name}Vector();
           </#if>
         </#list>
       </#list>
-      case STRUCT:
-        return getStruct().getObject(index);
-      case LIST:
-        return getList().getObject(index);
-      default:
-        throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]);
+        case STRUCT:
+          return getStruct();
+        case LIST:
+          return getList();
+        default:
+          throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]);
       }
     }
 
+    public Object getObject(int index) {
+      return getVector(index).getObject(index);
+    }
+
     public byte[] get(int index) {
       return null;
     }
@@ -622,4 +626,27 @@ public void setType(int index, MinorType type) {
     private int getTypeBufferValueCapacity() {
       return typeBuffer.capacity() / TYPE_WIDTH;
     }
+
+    @Override
+    public int hashCode(int index) {
+      return getVector(index).hashCode(index);
+    }
+
+    @Override
+    public boolean equals(int index, ValueVector to, int toIndex) {
+      if (to == null) {
+        return false;
+      }
+      if (this.getClass() != to.getClass()) {
+        return false;
+      }
+      UnionVector that = (UnionVector) to;
+      ValueVector leftVector = getVector(index);
+      ValueVector rightVector = that.getVector(toIndex);
+
+      if (leftVector.getClass() != rightVector.getClass()) {
+        return false;
+      }
+      return leftVector.equals(index, rightVector, toIndex);
+    }
 }

java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java

@@ -25,6 +25,7 @@
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.ByteFunctionHelpers;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
@@ -837,4 +838,33 @@ public void copyFromSafe(int fromIndex, int thisIndex, BaseFixedWidthVector from
     handleSafe(thisIndex);
     copyFrom(fromIndex, thisIndex, from);
   }
+
+  @Override
+  public int hashCode(int index) {
+    int start = typeWidth * index;
+    int end = typeWidth * (index + 1);
+    return ByteFunctionHelpers.hash(this.getDataBuffer(), start, end);
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    if (to == null) {
+      return false;
+    }
+    if (this.getClass() != to.getClass()) {
+      return false;
+    }
+
+    BaseFixedWidthVector that = (BaseFixedWidthVector) to;
+
+    int leftStart = typeWidth * index;
+    int leftEnd = typeWidth * (index + 1);
+
+    int rightStart = typeWidth * toIndex;
+    int rightEnd = typeWidth * (toIndex + 1);
+
+    int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
+        that.getDataBuffer(), rightStart, rightEnd);
+    return ret == 1;
+  }
 }

java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java

@@ -27,6 +27,7 @@
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.ByteFunctionHelpers;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.util.TransferPair;
@@ -1334,4 +1335,33 @@ public void copyFromSafe(int fromIndex, int thisIndex, BaseVariableWidthVector f
     }
     lastSet = thisIndex;
   }
+
+  @Override
+  public int hashCode(int index) {
+    final int start = getStartOffset(index);
+    final int end = getStartOffset(index + 1);
+    return ByteFunctionHelpers.hash(this.getDataBuffer(), start, end);
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    if (to == null) {
+      return false;
+    }
+    if (this.getClass() != to.getClass()) {
+      return false;
+    }
+
+    BaseVariableWidthVector that = (BaseVariableWidthVector) to;
+
+    final int leftStart = getStartOffset(index);
+    final int leftEnd = getStartOffset(index + 1);
+
+    final int rightStart = that.getStartOffset(toIndex);
+    final int rightEnd = that.getStartOffset(toIndex + 1);
+
+    int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
+        that.getDataBuffer(), rightStart, rightEnd);
+    return ret == 1;
+  }
 }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -237,4 +237,18 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    * @return true if element is null
    */
   boolean isNull(int index);
+
+  /**
+   * Returns hashCode of element in index.
+   */
+  int hashCode(int index);
+
+  /**
+   * Check whether the element in index equals to the element in targetIndex from the target vector.
+   * @param index index to compare in this vector
+   * @param target target vector
+   * @param targetIndex index to compare in target vector
+   * @return true if equals, otherwise false.
+   */
+  boolean equals(int index, ValueVector target, int targetIndex);
 }

java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java

@@ -270,7 +270,6 @@ public void setSafe(int index, Text text) {
     setSafe(index, text.getBytes(), 0, text.getLength());
   }
 
-
   /*----------------------------------------------------------------*
    |                                                                |
    |                      vector transfer                           |

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -244,4 +244,14 @@ public int getNullCount() {
   public boolean isNull(int index) {
     return false;
   }
+
+  @Override
+  public int hashCode(int index) {
+    return 0;
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    return false;
+  }
 }

java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

@@ -45,6 +45,7 @@
 import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.arrow.vector.util.ByteFunctionHelpers;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.OversizedAllocationException;
@@ -496,6 +497,37 @@ public TransferPair makeTransferPair(ValueVector target) {
     return new TransferImpl((FixedSizeListVector) target);
   }
 
+  @Override
+  public int hashCode(int index) {
+    if (isSet(index) == 0) {
+      return 0;
+    }
+    int hash = 0;
+    for (int i = 0; i < listSize; i++) {
+      hash = ByteFunctionHelpers.comebineHash(hash, vector.hashCode(index * listSize + i));
+    }
+    return hash;
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    if (to == null) {
+      return false;
+    }
+    if (this.getClass() != to.getClass()) {
+      return false;
+    }
+
+    FixedSizeListVector that = (FixedSizeListVector) to;
+
+    for (int i = 0; i < listSize; i++) {
+      if (!vector.equals(index * listSize + i, that, toIndex * listSize + i)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   private class TransferImpl implements TransferPair {
 
     FixedSizeListVector to;

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -412,6 +412,49 @@ public ArrowBuf getOffsetBuffer() {
     return offsetBuffer;
   }
 
+  @Override
+  public int hashCode(int index) {
+    if (isSet(index) == 0) {
+      return 0;
+    }
+    int hash = 0;
+    final int start = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    final int end = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+    for (int i = start; i < end; i++) {
+      hash = 31 * vector.hashCode(i);
+    }
+    return hash;
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    if (to == null) {
+      return false;
+    }
+    if (this.getClass() != to.getClass()) {
+      return false;
+    }
+
+    ListVector that = (ListVector) to;
+    final int leftStart = offsetBuffer.getInt(index * OFFSET_WIDTH);
+    final int leftEnd = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH);
+
+    final int rightStart = that.offsetBuffer.getInt(toIndex * OFFSET_WIDTH);
+    final int rightEnd = that.offsetBuffer.getInt((toIndex + 1) * OFFSET_WIDTH);
+
+    if ((leftEnd - leftStart) != (rightEnd - rightStart)) {
+      return false;
+    }
+
+    for (int i = 0; i < (leftEnd - leftStart); i++) {
+      if (!vector.equals(leftStart + i, that.vector, rightStart + i)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
   private class TransferImpl implements TransferPair {
 
     ListVector to;

java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java

@@ -286,6 +286,56 @@ public Object getObject(int index) {
     return vv;
   }
 
+  @Override
+  public int hashCode(int index) {
+    int hash = 0;
+    for (String child : getChildFieldNames()) {
+      ValueVector v = getChild(child);
+      if (v != null && index < v.getValueCount()) {
+        hash += 31 * hash + v.hashCode(index);
+      }
+    }
+    return hash;
+  }
+
+  @Override
+  public boolean equals(int index, ValueVector to, int toIndex) {
+    if (to == null) {
+      return false;
+    }
+    if (this.getClass() != to.getClass()) {
+      return false;
+    }
+    NonNullableStructVector that = (NonNullableStructVector) to;
+    List<ValueVector> leftChildrens = new ArrayList<>();
+    List<ValueVector> rightChildrens = new ArrayList<>();
+
+    for (String child : getChildFieldNames()) {
+      ValueVector v = getChild(child);
+      if (v != null) {
+        leftChildrens.add(v);
+      }
+    }
+
+    for (String child : that.getChildFieldNames()) {
+      ValueVector v = that.getChild(child);
+      if (v != null) {
+        rightChildrens.add(v);
+      }
+    }
+
+    if (leftChildrens.size() != rightChildrens.size()) {
+      return false;
+    }
+
+    for (int i = 0; i < leftChildrens.size(); i++) {
+      if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   @Override
   public boolean isNull(int index) {
     return false;
@@ -372,4 +422,5 @@ public void initializeChildrenFromFields(List<Field> children) {
   public List<FieldVector> getChildrenFromFields() {
     return getChildren();
   }
+
 }

java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java

@@ -479,6 +479,15 @@ public Object getObject(int index) {
     }
   }
 
+  @Override
+  public int hashCode(int index) {
+    if (isSet(index) == 0) {
+      return 0;
+    } else {
+      return super.hashCode(index);
+    }
+  }
+
   @Override
   public void get(int index, ComplexHolder holder) {
     holder.isSet = isSet(index);
@@ -546,4 +555,5 @@ public void setValueCount(int valueCount) {
     super.setValueCount(valueCount);
     this.valueCount = valueCount;
   }
+
 }