From 14b3e67edbc78355a9af5aadf2afaba85bff4212 Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 12 Aug 2019 17:51:09 +0800 Subject: [PATCH 1/4] ARROW-6210: [Java] remove equals API from ValueVector --- .../main/codegen/templates/UnionVector.java | 14 --- .../arrow/vector/BaseFixedWidthVector.java | 15 ---- .../arrow/vector/BaseVariableWidthVector.java | 14 --- .../org/apache/arrow/vector/ValueVector.java | 9 -- .../org/apache/arrow/vector/ZeroVector.java | 5 -- .../vector/compare/RangeEqualsVisitor.java | 86 ++++++++++++------- .../vector/complex/FixedSizeListVector.java | 14 --- .../arrow/vector/complex/ListVector.java | 15 ---- .../complex/NonNullableStructVector.java | 15 ---- .../dictionary/DictionaryHashTable.java | 7 +- .../apache/arrow/vector/TestValueVector.java | 4 +- .../vector/types/pojo/TestExtensionType.java | 6 -- 12 files changed, 66 insertions(+), 138 deletions(-) diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 6220e518cad..a809c508a3f 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -675,20 +675,6 @@ public int hashCode(int index) { return getVector(index).hashCode(index); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - @Override public boolean accept(RangeEqualsVisitor visitor) { return visitor.visit(this); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index a1663731fe3..eba70e8527c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -882,21 +882,6 @@ public int hashCode(int index) { return ByteFunctionHelpers.hash(this.getDataBuffer(), start, end); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - @Override public boolean accept(RangeEqualsVisitor visitor) { return visitor.visit(this); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index b7aa8161bd0..c5b3569a2c5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -1365,20 +1365,6 @@ public int hashCode(int index) { return ByteFunctionHelpers.hash(this.getDataBuffer(), start, end); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - @Override public boolean accept(RangeEqualsVisitor visitor) { return visitor.visit(this); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index aed24f259ee..af5cb651670 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -244,15 +244,6 @@ public interface ValueVector extends Closeable, Iterable { */ int hashCode(int index); - /** - * Check whether the element in index equals to the element in targetIndex from the target vector. - * @param index index to compare in this vector - * @param target target vector - * @param targetIndex index to compare in target vector - * @return true if equals, otherwise false. - */ - boolean equals(int index, ValueVector target, int targetIndex); - /** * Copy a cell value from a particular index in source vector to a particular * position in this vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 4a9f30aed47..d24d7ff4d34 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -251,11 +251,6 @@ public int hashCode(int index) { return 0; } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - return false; - } - @Override public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index a5826e7e48f..54155470d8d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.arrow.memory.util.ByteFunctionHelpers; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.FieldVector; @@ -37,9 +38,9 @@ public class RangeEqualsVisitor { protected final ValueVector right; - protected final int leftStart; - protected final int rightStart; - protected final int length; + protected int leftStart; + protected int rightStart; + protected int length; /** * Constructs a new instance. @@ -49,29 +50,56 @@ public RangeEqualsVisitor(ValueVector right, int leftStart, int rightStart, int this.rightStart = rightStart; this.right = right; this.length = length; + Preconditions.checkArgument(length >= 0, "length must be non negative"); + } + + /** + * Reset start indices and length for reuse purpose. + */ + public void reset(int leftStart, int rightStart, int length) { + this.leftStart = leftStart; + this.rightStart = rightStart; + this.length = length; + } + + private void validateIndices(ValueVector left) { + Preconditions.checkArgument(leftStart >= 0 && leftStart < left.getValueCount(), + "leftStart %s out of range[0, %s]:", 0, left.getValueCount()); + Preconditions.checkArgument((leftStart + length) <= left.getValueCount(), + "(leftStart + length) %s out of range[0, %s]:", 0, left.getValueCount()); + Preconditions.checkArgument(rightStart >= 0 && rightStart < right.getValueCount(), + "rightStart %s out of range[0, %s]:", 0, right.getValueCount()); + Preconditions.checkArgument((rightStart + length) <= right.getValueCount(), + "(rightStart + length) %s out of range[0, %s]:", 0, right.getValueCount()); } public boolean visit(BaseFixedWidthVector left) { + validateIndices(left); return compareBaseFixedWidthVectors(left); } public boolean visit(BaseVariableWidthVector left) { + validateIndices(left); return compareBaseVariableWidthVectors(left); } public boolean visit(ListVector left) { + validateIndices(left); return compareListVectors(left); } public boolean visit(FixedSizeListVector left) { + validateIndices(left); return compareFixedSizeListVectors(left); } public boolean visit(NonNullableStructVector left) { + validateIndices(left); return compareStructVectors(left); } public boolean visit(UnionVector left) { + validateIndices(left); return compareUnionVectors(left); } @@ -152,14 +180,14 @@ protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { int typeWidth = left.getTypeWidth(); if (!isNull) { - int startByteLeft = typeWidth * leftIndex; - int endByteLeft = typeWidth * (leftIndex + 1); + int startIndexLeft = typeWidth * leftIndex; + int endIndexLeft = typeWidth * (leftIndex + 1); - int startByteRight = typeWidth * rightIndex; - int endByteRight = typeWidth * (rightIndex + 1); + int startIndexRight = typeWidth * rightIndex; + int endIndexRight = typeWidth * (rightIndex + 1); - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, - right.getDataBuffer(), startByteRight, endByteRight); + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startIndexLeft, endIndexLeft, + right.getDataBuffer(), startIndexRight, endIndexRight); if (ret == 0) { return false; @@ -186,14 +214,14 @@ protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; if (!isNull) { - final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + final int startIndexLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endIndexLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + final int startIndexRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endIndexRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, - right.getDataBuffer(), startByteRight, endByteRight); + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startIndexLeft, endIndexLeft, + right.getDataBuffer(), startIndexRight, endIndexRight); if (ret == 0) { return false; @@ -220,21 +248,21 @@ protected boolean compareListVectors(ListVector left) { int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; if (!isNull) { - final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + final int startIndexLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endIndexLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + final int startIndexRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endIndexRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { + if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) { return false; } ValueVector leftDataVector = left.getDataVector(); ValueVector rightDataVector = ((ListVector)right).getDataVector(); - if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, - startByteRight, (endByteLeft - startByteLeft)))) { + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startIndexLeft, + startIndexRight, (endIndexLeft - startIndexLeft)))) { return false; } } @@ -263,21 +291,21 @@ protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { int listSize = left.getListSize(); if (!isNull) { - final int startByteLeft = leftIndex * listSize; - final int endByteLeft = (leftIndex + 1) * listSize; + final int startIndexLeft = leftIndex * listSize; + final int endIndexLeft = (leftIndex + 1) * listSize; - final int startByteRight = rightIndex * listSize; - final int endByteRight = (rightIndex + 1) * listSize; + final int startIndexRight = rightIndex * listSize; + final int endIndexRight = (rightIndex + 1) * listSize; - if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { + if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) { return false; } ValueVector leftDataVector = left.getDataVector(); ValueVector rightDataVector = ((FixedSizeListVector)right).getDataVector(); - if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, startByteRight, - (endByteLeft - startByteLeft)))) { + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startIndexLeft, startIndexRight, + (endIndexLeft - startIndexLeft)))) { return false; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 6bdf81753b7..bcebae68966 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -535,20 +535,6 @@ public int hashCode(int index) { return hash; } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - @Override public boolean accept(RangeEqualsVisitor visitor) { return visitor.visit(this); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index d6935dedb87..094b65874c9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -28,7 +28,6 @@ import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; -import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.BufferBacked; @@ -427,20 +426,6 @@ public int hashCode(int index) { return hash; } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - private class TransferImpl implements TransferPair { ListVector to; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 995751ed0b3..54cff432766 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; @@ -305,20 +304,6 @@ public boolean accept(RangeEqualsVisitor visitor) { return visitor.visit(this); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - Preconditions.checkArgument(index >= 0 && index < valueCount, - "index %s out of range[0, %s]:", index, valueCount - 1); - Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), - "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); - return this.accept(visitor); - } - @Override public boolean isNull(int index) { return false; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java index f6cf2d38416..5b813c10037 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java @@ -20,6 +20,7 @@ import java.util.Objects; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; /** * HashTable used for Dictionary encoding. It holds two vectors (the vector to encode and dictionary vector) @@ -74,6 +75,8 @@ public class DictionaryHashTable { private final ValueVector dictionary; + private final RangeEqualsVisitor visitor; + /** * Constructs an empty map with the specified initial capacity and load factor. */ @@ -94,6 +97,7 @@ public DictionaryHashTable(int initialCapacity, ValueVector dictionary) { for (int i = 0; i < this.dictionary.getValueCount(); i++) { put(i); } + this.visitor = new RangeEqualsVisitor(dictionary, 0, 0, 1); } public DictionaryHashTable(ValueVector dictionary) { @@ -140,7 +144,8 @@ public int getIndex(int indexInArray, ValueVector toEncode) { for (DictionaryHashTable.Entry e = table[index]; e != null ; e = e.next) { if ((e.hash == hash)) { int dictIndex = e.index; - if (dictionary.equals(dictIndex, toEncode, indexInArray)) { + visitor.reset(indexInArray, dictIndex, 1); + if (toEncode.accept(visitor)) { return dictIndex; } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index d344246fd59..36cfebac18c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -36,6 +36,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.compare.VectorEqualsVisitor; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.StructVector; @@ -2633,7 +2634,8 @@ public void testEqualsWithIndexOutOfRange() { vector2.setSafe(0, 1); vector2.setSafe(1, 2); - assertTrue(vector1.equals(3, vector2, 2)); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 3, 2, 1); + assertTrue(vector1.accept(visitor)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 792bd29903b..5f7522838d8 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -34,7 +34,6 @@ import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; @@ -210,11 +209,6 @@ public int hashCode(int index) { return getUnderlyingVector().hashCode(index); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - return getUnderlyingVector().equals(index, to, toIndex); - } - public void set(int index, UUID uuid) { ByteBuffer bb = ByteBuffer.allocate(16); bb.putLong(uuid.getMostSignificantBits()); From fb1510ec004b1b823c489db1703d3c0c9a77995d Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 12 Aug 2019 20:44:57 +0800 Subject: [PATCH 2/4] add ValueMatcher --- .../vector/compare/RangeEqualsVisitor.java | 55 +++++++------------ .../dictionary/DictionaryHashTable.java | 8 +-- .../arrow/vector/dictionary/ValueMatcher.java | 49 +++++++++++++++++ 3 files changed, 73 insertions(+), 39 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index 54155470d8d..3e3b73bbfd4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -42,6 +42,12 @@ public class RangeEqualsVisitor { protected int rightStart; protected int length; + protected boolean needCheckType = true; + + public void setNeedCheckType(boolean needCheckType) { + this.needCheckType = needCheckType; + } + /** * Constructs a new instance. */ @@ -62,7 +68,12 @@ public void reset(int leftStart, int rightStart, int length) { this.length = length; } - private void validateIndices(ValueVector left) { + private boolean validateTypesAndIndices(ValueVector left) { + + if (needCheckType && !compareValueVector(left, right)) { + return false; + } + Preconditions.checkArgument(leftStart >= 0 && leftStart < left.getValueCount(), "leftStart %s out of range[0, %s]:", 0, left.getValueCount()); Preconditions.checkArgument((leftStart + length) <= left.getValueCount(), @@ -71,36 +82,32 @@ private void validateIndices(ValueVector left) { "rightStart %s out of range[0, %s]:", 0, right.getValueCount()); Preconditions.checkArgument((rightStart + length) <= right.getValueCount(), "(rightStart + length) %s out of range[0, %s]:", 0, right.getValueCount()); + + return true; } public boolean visit(BaseFixedWidthVector left) { - validateIndices(left); - return compareBaseFixedWidthVectors(left); + return validateTypesAndIndices(left) && compareBaseFixedWidthVectors(left); } public boolean visit(BaseVariableWidthVector left) { - validateIndices(left); - return compareBaseVariableWidthVectors(left); + return validateTypesAndIndices(left) && compareBaseVariableWidthVectors(left); } public boolean visit(ListVector left) { - validateIndices(left); - return compareListVectors(left); + return validateTypesAndIndices(left) && compareListVectors(left); } public boolean visit(FixedSizeListVector left) { - validateIndices(left); - return compareFixedSizeListVectors(left); + return validateTypesAndIndices(left) && compareFixedSizeListVectors(left); } public boolean visit(NonNullableStructVector left) { - validateIndices(left); - return compareStructVectors(left); + return validateTypesAndIndices(left) && compareStructVectors(left); } public boolean visit(UnionVector left) { - validateIndices(left); - return compareUnionVectors(left); + return validateTypesAndIndices(left) && compareUnionVectors(left); } public boolean visit(ZeroVector left) { @@ -108,7 +115,7 @@ public boolean visit(ZeroVector left) { } public boolean visit(ValueVector left) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException(); } protected boolean compareValueVector(ValueVector left, ValueVector right) { @@ -117,10 +124,6 @@ protected boolean compareValueVector(ValueVector left, ValueVector right) { protected boolean compareUnionVectors(UnionVector left) { - if (!compareValueVector(left, right)) { - return false; - } - UnionVector rightVector = (UnionVector) right; List leftChildren = left.getChildrenFromFields(); @@ -141,9 +144,6 @@ protected boolean compareUnionVectors(UnionVector left) { } protected boolean compareStructVectors(NonNullableStructVector left) { - if (!compareValueVector(left, right)) { - return false; - } NonNullableStructVector rightVector = (NonNullableStructVector) right; @@ -164,10 +164,6 @@ protected boolean compareStructVectors(NonNullableStructVector left) { protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { - if (!compareValueVector(left, right)) { - return false; - } - for (int i = 0; i < length; i++) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; @@ -198,9 +194,6 @@ protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { } protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) { - if (!compareValueVector(left, right)) { - return false; - } for (int i = 0; i < length; i++) { int leftIndex = leftStart + i; @@ -232,9 +225,6 @@ protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) } protected boolean compareListVectors(ListVector left) { - if (!compareValueVector(left, right)) { - return false; - } for (int i = 0; i < length; i++) { int leftIndex = leftStart + i; @@ -271,9 +261,6 @@ protected boolean compareListVectors(ListVector left) { } protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { - if (!compareValueVector(left, right)) { - return false; - } if (left.getListSize() != ((FixedSizeListVector)right).getListSize()) { return false; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java index 5b813c10037..1cee5841278 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java @@ -20,7 +20,6 @@ import java.util.Objects; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.compare.RangeEqualsVisitor; /** * HashTable used for Dictionary encoding. It holds two vectors (the vector to encode and dictionary vector) @@ -75,7 +74,7 @@ public class DictionaryHashTable { private final ValueVector dictionary; - private final RangeEqualsVisitor visitor; + private final ValueMatcher matcher; /** * Constructs an empty map with the specified initial capacity and load factor. @@ -97,7 +96,7 @@ public DictionaryHashTable(int initialCapacity, ValueVector dictionary) { for (int i = 0; i < this.dictionary.getValueCount(); i++) { put(i); } - this.visitor = new RangeEqualsVisitor(dictionary, 0, 0, 1); + this.matcher = new ValueMatcher(dictionary); } public DictionaryHashTable(ValueVector dictionary) { @@ -144,8 +143,7 @@ public int getIndex(int indexInArray, ValueVector toEncode) { for (DictionaryHashTable.Entry e = table[index]; e != null ; e = e.next) { if ((e.hash == hash)) { int dictIndex = e.index; - visitor.reset(indexInArray, dictIndex, 1); - if (toEncode.accept(visitor)) { + if (matcher.match(dictIndex, toEncode, indexInArray)) { return dictIndex; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java new file mode 100644 index 00000000000..b204f9cb51a --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.dictionary; + +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; + +/** + * A special {@link RangeEqualsVisitor} used to compare single values + * which used in dictionary encoding. + */ +public class ValueMatcher { + + private RangeEqualsVisitor visitor; + + /** + * Constructs a new instance. + */ + public ValueMatcher(ValueVector dictionary) { + this.visitor = new RangeEqualsVisitor(dictionary, 0, 0, 1); + visitor.setNeedCheckType(false); + } + + /** + * Check if single value equals in the two vectors. + * @param dictIndex index in dictionary + * @param index index in the vector to encode. + * @return true if value matches, otherwise false + */ + public boolean match(int dictIndex, ValueVector vector, int index) { + visitor.reset(index, dictIndex, 1); + return vector.accept(visitor); + } +} From ad2277b738fa4bb66877b99ab5a6d4a9624ad1b7 Mon Sep 17 00:00:00 2001 From: tianchen Date: Tue, 13 Aug 2019 16:33:33 +0800 Subject: [PATCH 3/4] resolve comments and port tests --- .../vector/compare/RangeEqualsVisitor.java | 56 +- .../vector/compare/VectorEqualsVisitor.java | 6 +- .../dictionary/DictionaryHashTable.java | 6 +- .../arrow/vector/dictionary/ValueMatcher.java | 49 -- .../apache/arrow/vector/TestValueVector.java | 409 --------------- .../compare/TestRangeEqualsVisitor.java | 478 ++++++++++++++++++ 6 files changed, 515 insertions(+), 489 deletions(-) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index 3e3b73bbfd4..1b2bda94fbe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -42,83 +42,88 @@ public class RangeEqualsVisitor { protected int rightStart; protected int length; - protected boolean needCheckType = true; + protected boolean typeCheckNeeded = true; - public void setNeedCheckType(boolean needCheckType) { - this.needCheckType = needCheckType; + public void setTypeCheckNeeded(boolean typeCheckNeeded) { + this.typeCheckNeeded = typeCheckNeeded; } /** * Constructs a new instance. */ - public RangeEqualsVisitor(ValueVector right, int leftStart, int rightStart, int length) { + public RangeEqualsVisitor(ValueVector right, int rightStart, int leftStart, int length, boolean typeCheckNeeded) { this.leftStart = leftStart; this.rightStart = rightStart; this.right = right; this.length = length; + this.typeCheckNeeded = true; Preconditions.checkArgument(length >= 0, "length must be non negative"); } /** - * Reset start indices and length for reuse purpose. + * Constructs a new instance. */ - public void reset(int leftStart, int rightStart, int length) { - this.leftStart = leftStart; - this.rightStart = rightStart; - this.length = length; + public RangeEqualsVisitor(ValueVector right, int leftStart, int rightStart, int length) { + this (right, rightStart, leftStart, length, true); } - private boolean validateTypesAndIndices(ValueVector left) { + /** + * Do some validation work, like type check and indices check. + */ + private boolean validate(ValueVector left) { - if (needCheckType && !compareValueVector(left, right)) { + if (!compareValueVector(left, right)) { return false; } - Preconditions.checkArgument(leftStart >= 0 && leftStart < left.getValueCount(), - "leftStart %s out of range[0, %s]:", 0, left.getValueCount()); + Preconditions.checkArgument(leftStart >= 0, + "leftStart %s must be non negative.", leftStart); Preconditions.checkArgument((leftStart + length) <= left.getValueCount(), - "(leftStart + length) %s out of range[0, %s]:", 0, left.getValueCount()); - Preconditions.checkArgument(rightStart >= 0 && rightStart < right.getValueCount(), - "rightStart %s out of range[0, %s]:", 0, right.getValueCount()); + "(leftStart + length) %s out of range[0, %s].", 0, left.getValueCount()); + Preconditions.checkArgument(rightStart >= 0, + "rightStart %s must be non negative.", rightStart); Preconditions.checkArgument((rightStart + length) <= right.getValueCount(), - "(rightStart + length) %s out of range[0, %s]:", 0, right.getValueCount()); + "(rightStart + length) %s out of range[0, %s].", 0, right.getValueCount()); return true; } public boolean visit(BaseFixedWidthVector left) { - return validateTypesAndIndices(left) && compareBaseFixedWidthVectors(left); + return validate(left) && compareBaseFixedWidthVectors(left); } public boolean visit(BaseVariableWidthVector left) { - return validateTypesAndIndices(left) && compareBaseVariableWidthVectors(left); + return validate(left) && compareBaseVariableWidthVectors(left); } public boolean visit(ListVector left) { - return validateTypesAndIndices(left) && compareListVectors(left); + return validate(left) && compareListVectors(left); } public boolean visit(FixedSizeListVector left) { - return validateTypesAndIndices(left) && compareFixedSizeListVectors(left); + return validate(left) && compareFixedSizeListVectors(left); } public boolean visit(NonNullableStructVector left) { - return validateTypesAndIndices(left) && compareStructVectors(left); + return validate(left) && compareStructVectors(left); } public boolean visit(UnionVector left) { - return validateTypesAndIndices(left) && compareUnionVectors(left); + return validate(left) && compareUnionVectors(left); } public boolean visit(ZeroVector left) { - return compareValueVector(left, right); + return validate(left); } public boolean visit(ValueVector left) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException(); } protected boolean compareValueVector(ValueVector left, ValueVector right) { + if (!typeCheckNeeded) { + return true; + } return left.getField().getType().equals(right.getField().getType()); } @@ -299,5 +304,4 @@ protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { } return true; } - } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java index 47071dd1958..dfaf45fa43f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java @@ -26,7 +26,11 @@ public class VectorEqualsVisitor extends RangeEqualsVisitor { public VectorEqualsVisitor(ValueVector right) { - super(Preconditions.checkNotNull(right), 0, 0, right.getValueCount()); + this(right, true); + } + + public VectorEqualsVisitor(ValueVector right, boolean typeCheckNeeded) { + super(Preconditions.checkNotNull(right), 0, 0, right.getValueCount(), typeCheckNeeded); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java index 1cee5841278..e7d0727a3bd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java @@ -20,6 +20,7 @@ import java.util.Objects; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; /** * HashTable used for Dictionary encoding. It holds two vectors (the vector to encode and dictionary vector) @@ -74,8 +75,6 @@ public class DictionaryHashTable { private final ValueVector dictionary; - private final ValueMatcher matcher; - /** * Constructs an empty map with the specified initial capacity and load factor. */ @@ -96,7 +95,6 @@ public DictionaryHashTable(int initialCapacity, ValueVector dictionary) { for (int i = 0; i < this.dictionary.getValueCount(); i++) { put(i); } - this.matcher = new ValueMatcher(dictionary); } public DictionaryHashTable(ValueVector dictionary) { @@ -143,7 +141,7 @@ public int getIndex(int indexInArray, ValueVector toEncode) { for (DictionaryHashTable.Entry e = table[index]; e != null ; e = e.next) { if ((e.hash == hash)) { int dictIndex = e.index; - if (matcher.match(dictIndex, toEncode, indexInArray)) { + if (toEncode.accept(new RangeEqualsVisitor(dictionary, dictIndex, indexInArray, 1, false))) { return dictIndex; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java deleted file mode 100644 index b204f9cb51a..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/ValueMatcher.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.dictionary; - -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.compare.RangeEqualsVisitor; - -/** - * A special {@link RangeEqualsVisitor} used to compare single values - * which used in dictionary encoding. - */ -public class ValueMatcher { - - private RangeEqualsVisitor visitor; - - /** - * Constructs a new instance. - */ - public ValueMatcher(ValueVector dictionary) { - this.visitor = new RangeEqualsVisitor(dictionary, 0, 0, 1); - visitor.setNeedCheckType(false); - } - - /** - * Check if single value equals in the two vectors. - * @param dictIndex index in dictionary - * @param index index in the vector to encode. - * @return true if value matches, otherwise false - */ - public boolean match(int dictIndex, ValueVector vector, int index) { - visitor.reset(index, dictIndex, 1); - return vector.accept(visitor); - } -} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 36cfebac18c..15f6910256c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -36,23 +36,12 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; -import org.apache.arrow.vector.compare.RangeEqualsVisitor; -import org.apache.arrow.vector.compare.VectorEqualsVisitor; -import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.StructVector; -import org.apache.arrow.vector.complex.UnionVector; -import org.apache.arrow.vector.complex.impl.NullableStructWriter; -import org.apache.arrow.vector.complex.impl.UnionListWriter; -import org.apache.arrow.vector.holders.NullableIntHolder; -import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; -import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.Text; @@ -2255,402 +2244,4 @@ public void testGetNullFromVariableWidthVector() { assertNull(varBinaryVector.get(0)); } } - - @Test - public void testZeroVectorEquals() { - try (final ZeroVector vector1 = new ZeroVector(); - final ZeroVector vector2 = new ZeroVector()) { - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testZeroVectorNotEquals() { - try (final IntVector intVector = new IntVector("int", allocator); - final ZeroVector zeroVector = new ZeroVector()) { - - VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); - assertFalse(intVector.accept(zeroVisitor)); - - VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); - assertFalse(zeroVector.accept(intVisitor)); - } - } - - @Test - public void testIntVectorEqualsWithNull() { - try (final IntVector vector1 = new IntVector("int", allocator); - final IntVector vector2 = new IntVector("int", allocator)) { - - vector1.allocateNew(2); - vector1.setValueCount(2); - vector2.allocateNew(2); - vector2.setValueCount(2); - - vector1.setSafe(0, 1); - vector1.setSafe(1, 2); - - vector2.setSafe(0, 1); - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testIntVectorEquals() { - try (final IntVector vector1 = new IntVector("int", allocator); - final IntVector vector2 = new IntVector("int", allocator)) { - - vector1.allocateNew(3); - vector1.setValueCount(3); - vector2.allocateNew(3); - vector2.setValueCount(2); - - vector1.setSafe(0, 1); - vector1.setSafe(1, 2); - vector1.setSafe(2, 3); - - vector2.setSafe(0, 1); - vector2.setSafe(1, 2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - - assertFalse(vector1.accept(visitor)); - - vector2.setValueCount(3); - vector2.setSafe(2, 2); - assertFalse(vector1.equals(vector2)); - - vector2.setSafe(2, 3); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testDecimalVectorEquals() { - try (final DecimalVector vector1 = new DecimalVector("decimal", allocator, 3, 3); - final DecimalVector vector2 = new DecimalVector("decimal", allocator, 3, 3); - final DecimalVector vector3 = new DecimalVector("decimal", allocator, 3, 2)) { - - vector1.allocateNew(2); - vector1.setValueCount(2); - vector2.allocateNew(2); - vector2.setValueCount(2); - vector3.allocateNew(2); - vector3.setValueCount(2); - - vector1.setSafe(0, 100); - vector1.setSafe(1, 200); - - vector2.setSafe(0, 100); - vector2.setSafe(1, 200); - - vector3.setSafe(0, 100); - vector3.setSafe(1, 200); - - VectorEqualsVisitor visitor1 = new VectorEqualsVisitor(vector2); - VectorEqualsVisitor visitor2 = new VectorEqualsVisitor(vector3); - - assertTrue(vector1.accept(visitor1)); - assertFalse(vector1.accept(visitor2)); - } - } - - @Test - public void testVarcharVectorEuqalsWithNull() { - try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); - final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { - - vector1.allocateNew(); - vector2.allocateNew(); - - // set some values - vector1.setSafe(0, STR1, 0, STR1.length); - vector1.setSafe(1, STR2, 0, STR2.length); - vector1.setValueCount(2); - - vector2.setSafe(0, STR1, 0, STR1.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testVarcharVectorEquals() { - try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); - final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { - - vector1.allocateNew(); - vector2.allocateNew(); - - // set some values - vector1.setSafe(0, STR1, 0, STR1.length); - vector1.setSafe(1, STR2, 0, STR2.length); - vector1.setSafe(2, STR3, 0, STR3.length); - vector1.setValueCount(3); - - vector2.setSafe(0, STR1, 0, STR1.length); - vector2.setSafe(1, STR2, 0, STR2.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - - vector2.setSafe(2, STR3, 0, STR3.length); - vector2.setValueCount(3); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testVarBinaryVectorEquals() { - try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); - final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { - - vector1.allocateNew(); - vector2.allocateNew(); - - // set some values - vector1.setSafe(0, STR1, 0, STR1.length); - vector1.setSafe(1, STR2, 0, STR2.length); - vector1.setSafe(2, STR3, 0, STR3.length); - vector1.setValueCount(3); - - vector2.setSafe(0, STR1, 0, STR1.length); - vector2.setSafe(1, STR2, 0, STR2.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - - vector2.setSafe(2, STR3, 0, STR3.length); - vector2.setValueCount(3); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testListVectorEqualsWithNull() { - try (final ListVector vector1 = ListVector.empty("list", allocator); - final ListVector vector2 = ListVector.empty("list", allocator);) { - - UnionListWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - //set some values - writeListVector(writer1, new int[] {1, 2}); - writeListVector(writer1, new int[] {3, 4}); - writeListVector(writer1, new int[] {}); - writer1.setValueCount(3); - - UnionListWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - //set some values - writeListVector(writer2, new int[] {1, 2}); - writeListVector(writer2, new int[] {3, 4}); - writer2.setValueCount(3); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testListVectorEquals() { - try (final ListVector vector1 = ListVector.empty("list", allocator); - final ListVector vector2 = ListVector.empty("list", allocator);) { - - UnionListWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - //set some values - writeListVector(writer1, new int[] {1, 2}); - writeListVector(writer1, new int[] {3, 4}); - writeListVector(writer1, new int[] {5, 6}); - writer1.setValueCount(3); - - UnionListWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - //set some values - writeListVector(writer2, new int[] {1, 2}); - writeListVector(writer2, new int[] {3, 4}); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - - writeListVector(writer2, new int[] {5, 6}); - writer2.setValueCount(3); - - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testStructVectorEqualsWithNull() { - - try (final StructVector vector1 = StructVector.empty("struct", allocator); - final StructVector vector2 = StructVector.empty("struct", allocator);) { - vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - - NullableStructWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - writeStructVector(writer1, 1, 10L); - writeStructVector(writer1, 2, 20L); - writeStructVector(writer1, 3, 30L); - writer1.setValueCount(3); - - NullableStructWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - writeStructVector(writer2, 1, 10L); - writeStructVector(writer2, 3, 30L); - writer2.setValueCount(3); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testStructVectorEquals() { - try (final StructVector vector1 = StructVector.empty("struct", allocator); - final StructVector vector2 = StructVector.empty("struct", allocator);) { - vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - - NullableStructWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - writeStructVector(writer1, 1, 10L); - writeStructVector(writer1, 2, 20L); - writeStructVector(writer1, 3, 30L); - writer1.setValueCount(3); - - NullableStructWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - writeStructVector(writer2, 1, 10L); - writeStructVector(writer2, 2, 20L); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - - writeStructVector(writer2, 3, 30L); - writer2.setValueCount(3); - - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testStructVectorEqualsWithDiffChild() { - try (final StructVector vector1 = StructVector.empty("struct", allocator); - final StructVector vector2 = StructVector.empty("struct", allocator);) { - vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector2.addOrGet("f10", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - - NullableStructWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - writeStructVector(writer1, 1, 10L); - writeStructVector(writer1, 2, 20L); - writer1.setValueCount(2); - - NullableStructWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - writeStructVector(writer2, 1, 10L); - writeStructVector(writer2, 2, 20L); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testUnionVectorEquals() { - try (final UnionVector vector1 = new UnionVector("union", allocator, null); - final UnionVector vector2 = new UnionVector("union", allocator, null);) { - - final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); - uInt4Holder.value = 10; - uInt4Holder.isSet = 1; - - final NullableIntHolder intHolder = new NullableIntHolder(); - uInt4Holder.value = 20; - uInt4Holder.isSet = 1; - - vector1.setType(0, Types.MinorType.UINT4); - vector1.setSafe(0, uInt4Holder); - - vector1.setType(1, Types.MinorType.INT); - vector1.setSafe(1, intHolder); - vector1.setValueCount(2); - - vector2.setType(0, Types.MinorType.UINT4); - vector2.setSafe(0, uInt4Holder); - - vector2.setType(1, Types.MinorType.INT); - vector2.setSafe(1, intHolder); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertTrue(vector1.accept(visitor)); - } - } - - @Test(expected = IllegalArgumentException.class) - public void testEqualsWithIndexOutOfRange() { - try (final IntVector vector1 = new IntVector("int", allocator); - final IntVector vector2 = new IntVector("int", allocator)) { - - vector1.allocateNew(2); - vector1.setValueCount(2); - vector2.allocateNew(2); - vector2.setValueCount(2); - - vector1.setSafe(0, 1); - vector1.setSafe(1, 2); - - vector2.setSafe(0, 1); - vector2.setSafe(1, 2); - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 3, 2, 1); - assertTrue(vector1.accept(visitor)); - } - } - - private void writeStructVector(NullableStructWriter writer, int value1, long value2) { - writer.start(); - writer.integer("f0").writeInt(value1); - writer.bigInt("f1").writeBigInt(value2); - writer.end(); - } - - private void writeListVector(UnionListWriter writer, int[] values) { - writer.startList(); - for (int v: values) { - writer.integer().writeInt(v); - } - writer.endList(); - } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java new file mode 100644 index 00000000000..721d5d9bf6b --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compare; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.Charset; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.NullableStructWriter; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestRangeEqualsVisitor { + + private BufferAllocator allocator; + + @Before + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + private static final Charset utf8Charset = Charset.forName("UTF-8"); + private static final byte[] STR1 = "AAAAA1".getBytes(utf8Charset); + private static final byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset); + private static final byte[] STR3 = "CCCC3".getBytes(utf8Charset); + + @After + public void terminate() throws Exception { + allocator.close(); + } + + @Test + public void testZeroVectorEquals() { + try (final ZeroVector vector1 = new ZeroVector(); + final ZeroVector vector2 = new ZeroVector()) { + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testZeroVectorNotEquals() { + try (final IntVector intVector = new IntVector("int", allocator); + final ZeroVector zeroVector = new ZeroVector()) { + + VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); + assertFalse(intVector.accept(zeroVisitor)); + + VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); + assertFalse(zeroVector.accept(intVisitor)); + } + } + + @Test + public void testIntVectorEqualsWithNull() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testIntVectorEquals() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(3); + vector1.setValueCount(3); + vector2.allocateNew(3); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + vector1.setSafe(2, 3); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + + vector2.setValueCount(3); + vector2.setSafe(2, 2); + assertFalse(vector1.equals(vector2)); + + vector2.setSafe(2, 3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testDecimalVectorEquals() { + try (final DecimalVector vector1 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector2 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector3 = new DecimalVector("decimal", allocator, 3, 2)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + vector3.allocateNew(2); + vector3.setValueCount(2); + + vector1.setSafe(0, 100); + vector1.setSafe(1, 200); + + vector2.setSafe(0, 100); + vector2.setSafe(1, 200); + + vector3.setSafe(0, 100); + vector3.setSafe(1, 200); + + VectorEqualsVisitor visitor1 = new VectorEqualsVisitor(vector2); + VectorEqualsVisitor visitor2 = new VectorEqualsVisitor(vector3); + + assertTrue(vector1.accept(visitor1)); + assertFalse(vector1.accept(visitor2)); + } + } + + @Test + public void testVarcharVectorEuqalsWithNull() { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setValueCount(2); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testVarcharVectorEquals() { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testVarBinaryVectorEquals() { + try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testListVectorEqualsWithNull() { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(3); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testListVectorEquals() { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {5, 6}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + writeListVector(writer2, new int[] {5, 6}); + writer2.setValueCount(3); + + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEqualsWithNull() { + + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEquals() { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEqualsWithDiffChild() { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f10", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writer1.setValueCount(2); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testUnionVectorEquals() { + try (final UnionVector vector1 = new UnionVector("union", allocator, null); + final UnionVector vector2 = new UnionVector("union", allocator, null);) { + + final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); + uInt4Holder.value = 10; + uInt4Holder.isSet = 1; + + final NullableIntHolder intHolder = new NullableIntHolder(); + uInt4Holder.value = 20; + uInt4Holder.isSet = 1; + + vector1.setType(0, Types.MinorType.UINT4); + vector1.setSafe(0, uInt4Holder); + + vector1.setType(1, Types.MinorType.INT); + vector1.setSafe(1, intHolder); + vector1.setValueCount(2); + + vector2.setType(0, Types.MinorType.UINT4); + vector2.setSafe(0, uInt4Holder); + + vector2.setType(1, Types.MinorType.INT); + vector2.setSafe(1, intHolder); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testEqualsWithIndexOutOfRange() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 3, 2, 1); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testEqualsWithOutTypeCheck() { + try (final IntVector intVector = new IntVector("int", allocator); + final ZeroVector zeroVector = new ZeroVector()) { + + VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); + zeroVisitor.setTypeCheckNeeded(false); + assertTrue(intVector.accept(zeroVisitor)); + + VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); + intVisitor.setTypeCheckNeeded(false); + assertTrue(zeroVector.accept(intVisitor)); + } + } + + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { + writer.start(); + writer.integer("f0").writeInt(value1); + writer.bigInt("f1").writeBigInt(value2); + writer.end(); + } + + private void writeListVector(UnionListWriter writer, int[] values) { + writer.startList(); + for (int v: values) { + writer.integer().writeInt(v); + } + writer.endList(); + } +} From cae440a755ef6302f52e9950b9c4d19d1974e286 Mon Sep 17 00:00:00 2001 From: tianchen Date: Wed, 14 Aug 2019 08:01:50 +0800 Subject: [PATCH 4/4] resolve comments --- .../vector/compare/RangeEqualsVisitor.java | 8 +- .../apache/arrow/vector/TestValueVector.java | 409 ++++++++++++++++++ .../compare/TestRangeEqualsVisitor.java | 306 +++---------- 3 files changed, 460 insertions(+), 263 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index 1b2bda94fbe..537746eff6c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -44,10 +44,6 @@ public class RangeEqualsVisitor { protected boolean typeCheckNeeded = true; - public void setTypeCheckNeeded(boolean typeCheckNeeded) { - this.typeCheckNeeded = typeCheckNeeded; - } - /** * Constructs a new instance. */ @@ -56,7 +52,7 @@ public RangeEqualsVisitor(ValueVector right, int rightStart, int leftStart, int this.rightStart = rightStart; this.right = right; this.length = length; - this.typeCheckNeeded = true; + this.typeCheckNeeded = typeCheckNeeded; Preconditions.checkArgument(length >= 0, "length must be non negative"); } @@ -64,7 +60,7 @@ public RangeEqualsVisitor(ValueVector right, int rightStart, int leftStart, int * Constructs a new instance. */ public RangeEqualsVisitor(ValueVector right, int leftStart, int rightStart, int length) { - this (right, rightStart, leftStart, length, true); + this(right, rightStart, leftStart, length, true); } /** diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 15f6910256c..3092492d7ac 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -36,12 +36,23 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; +import org.apache.arrow.vector.compare.VectorEqualsVisitor; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.NullableStructWriter; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.Text; @@ -2244,4 +2255,402 @@ public void testGetNullFromVariableWidthVector() { assertNull(varBinaryVector.get(0)); } } + + @Test + public void testZeroVectorEquals() { + try (final ZeroVector vector1 = new ZeroVector(); + final ZeroVector vector2 = new ZeroVector()) { + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testZeroVectorNotEquals() { + try (final IntVector intVector = new IntVector("int", allocator); + final ZeroVector zeroVector = new ZeroVector()) { + + VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); + assertFalse(intVector.accept(zeroVisitor)); + + VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); + assertFalse(zeroVector.accept(intVisitor)); + } + } + + @Test + public void testIntVectorEqualsWithNull() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testIntVectorEquals() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(3); + vector1.setValueCount(3); + vector2.allocateNew(3); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + vector1.setSafe(2, 3); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + + vector2.setValueCount(3); + vector2.setSafe(2, 2); + assertFalse(vector1.equals(vector2)); + + vector2.setSafe(2, 3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testDecimalVectorEquals() { + try (final DecimalVector vector1 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector2 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector3 = new DecimalVector("decimal", allocator, 3, 2)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + vector3.allocateNew(2); + vector3.setValueCount(2); + + vector1.setSafe(0, 100); + vector1.setSafe(1, 200); + + vector2.setSafe(0, 100); + vector2.setSafe(1, 200); + + vector3.setSafe(0, 100); + vector3.setSafe(1, 200); + + VectorEqualsVisitor visitor1 = new VectorEqualsVisitor(vector2); + VectorEqualsVisitor visitor2 = new VectorEqualsVisitor(vector3); + + assertTrue(vector1.accept(visitor1)); + assertFalse(vector1.accept(visitor2)); + } + } + + @Test + public void testVarcharVectorEuqalsWithNull() { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setValueCount(2); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testVarcharVectorEquals() { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testVarBinaryVectorEquals() { + try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testListVectorEqualsWithNull() { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(3); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testListVectorEquals() { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {5, 6}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + writeListVector(writer2, new int[] {5, 6}); + writer2.setValueCount(3); + + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEqualsWithNull() { + + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEquals() { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + assertTrue(vector1.accept(visitor)); + } + } + + @Test + public void testStructVectorEqualsWithDiffChild() { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f10", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writer1.setValueCount(2); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); + } + } + + @Test + public void testUnionVectorEquals() { + try (final UnionVector vector1 = new UnionVector("union", allocator, null); + final UnionVector vector2 = new UnionVector("union", allocator, null);) { + + final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); + uInt4Holder.value = 10; + uInt4Holder.isSet = 1; + + final NullableIntHolder intHolder = new NullableIntHolder(); + uInt4Holder.value = 20; + uInt4Holder.isSet = 1; + + vector1.setType(0, Types.MinorType.UINT4); + vector1.setSafe(0, uInt4Holder); + + vector1.setType(1, Types.MinorType.INT); + vector1.setSafe(1, intHolder); + vector1.setValueCount(2); + + vector2.setType(0, Types.MinorType.UINT4); + vector2.setSafe(0, uInt4Holder); + + vector2.setType(1, Types.MinorType.INT); + vector2.setSafe(1, intHolder); + vector2.setValueCount(2); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testEqualsWithIndexOutOfRange() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 3, 2, 1); + assertTrue(vector1.accept(visitor)); + } + } + + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { + writer.start(); + writer.integer("f0").writeInt(value1); + writer.bigInt("f1").writeBigInt(value2); + writer.end(); + } + + private void writeListVector(UnionListWriter writer, int[] values) { + writer.startList(); + for (int v: values) { + writer.integer().writeInt(v); + } + writer.endList(); + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java index 721d5d9bf6b..847da35d56c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java @@ -25,9 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.ListVector; @@ -63,29 +61,6 @@ public void terminate() throws Exception { allocator.close(); } - @Test - public void testZeroVectorEquals() { - try (final ZeroVector vector1 = new ZeroVector(); - final ZeroVector vector2 = new ZeroVector()) { - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testZeroVectorNotEquals() { - try (final IntVector intVector = new IntVector("int", allocator); - final ZeroVector zeroVector = new ZeroVector()) { - - VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); - assertFalse(intVector.accept(zeroVisitor)); - - VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); - assertFalse(zeroVector.accept(intVisitor)); - } - } - @Test public void testIntVectorEqualsWithNull() { try (final IntVector vector1 = new IntVector("int", allocator); @@ -107,88 +82,34 @@ public void testIntVectorEqualsWithNull() { } @Test - public void testIntVectorEquals() { + public void testBaseFixedWidthVectorRangeEqual() { try (final IntVector vector1 = new IntVector("int", allocator); final IntVector vector2 = new IntVector("int", allocator)) { - vector1.allocateNew(3); - vector1.setValueCount(3); - vector2.allocateNew(3); - vector2.setValueCount(2); + vector1.allocateNew(5); + vector1.setValueCount(5); + vector2.allocateNew(5); + vector2.setValueCount(5); vector1.setSafe(0, 1); vector1.setSafe(1, 2); vector1.setSafe(2, 3); + vector1.setSafe(3, 4); + vector1.setSafe(4, 5); - vector2.setSafe(0, 1); + vector2.setSafe(0, 11); vector2.setSafe(1, 2); + vector2.setSafe(2,3); + vector2.setSafe(3,4); + vector2.setSafe(4,55); - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - - assertFalse(vector1.accept(visitor)); - - vector2.setValueCount(3); - vector2.setSafe(2, 2); - assertFalse(vector1.equals(vector2)); - - vector2.setSafe(2, 3); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 1, 1, 3); assertTrue(vector1.accept(visitor)); } } @Test - public void testDecimalVectorEquals() { - try (final DecimalVector vector1 = new DecimalVector("decimal", allocator, 3, 3); - final DecimalVector vector2 = new DecimalVector("decimal", allocator, 3, 3); - final DecimalVector vector3 = new DecimalVector("decimal", allocator, 3, 2)) { - - vector1.allocateNew(2); - vector1.setValueCount(2); - vector2.allocateNew(2); - vector2.setValueCount(2); - vector3.allocateNew(2); - vector3.setValueCount(2); - - vector1.setSafe(0, 100); - vector1.setSafe(1, 200); - - vector2.setSafe(0, 100); - vector2.setSafe(1, 200); - - vector3.setSafe(0, 100); - vector3.setSafe(1, 200); - - VectorEqualsVisitor visitor1 = new VectorEqualsVisitor(vector2); - VectorEqualsVisitor visitor2 = new VectorEqualsVisitor(vector3); - - assertTrue(vector1.accept(visitor1)); - assertFalse(vector1.accept(visitor2)); - } - } - - @Test - public void testVarcharVectorEuqalsWithNull() { - try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); - final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { - - vector1.allocateNew(); - vector2.allocateNew(); - - // set some values - vector1.setSafe(0, STR1, 0, STR1.length); - vector1.setSafe(1, STR2, 0, STR2.length); - vector1.setValueCount(2); - - vector2.setSafe(0, STR1, 0, STR1.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testVarcharVectorEquals() { + public void testBaseVariableVectorRangeEquals() { try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { @@ -199,78 +120,24 @@ public void testVarcharVectorEquals() { vector1.setSafe(0, STR1, 0, STR1.length); vector1.setSafe(1, STR2, 0, STR2.length); vector1.setSafe(2, STR3, 0, STR3.length); - vector1.setValueCount(3); + vector1.setSafe(3, STR2, 0, STR2.length); + vector1.setSafe(4, STR1, 0, STR1.length); + vector1.setValueCount(5); vector2.setSafe(0, STR1, 0, STR1.length); vector2.setSafe(1, STR2, 0, STR2.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - vector2.setSafe(2, STR3, 0, STR3.length); - vector2.setValueCount(3); - assertTrue(vector1.accept(visitor)); - } - } - - @Test - public void testVarBinaryVectorEquals() { - try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); - final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { - - vector1.allocateNew(); - vector2.allocateNew(); - - // set some values - vector1.setSafe(0, STR1, 0, STR1.length); - vector1.setSafe(1, STR2, 0, STR2.length); - vector1.setSafe(2, STR3, 0, STR3.length); - vector1.setValueCount(3); - - vector2.setSafe(0, STR1, 0, STR1.length); - vector2.setSafe(1, STR2, 0, STR2.length); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); + vector2.setSafe(3, STR2, 0, STR2.length); + vector2.setSafe(4, STR1, 0, STR1.length); + vector2.setValueCount(5); - vector2.setSafe(2, STR3, 0, STR3.length); - vector2.setValueCount(3); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 1, 1, 3); assertTrue(vector1.accept(visitor)); } } @Test - public void testListVectorEqualsWithNull() { - try (final ListVector vector1 = ListVector.empty("list", allocator); - final ListVector vector2 = ListVector.empty("list", allocator);) { - - UnionListWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - //set some values - writeListVector(writer1, new int[] {1, 2}); - writeListVector(writer1, new int[] {3, 4}); - writeListVector(writer1, new int[] {}); - writer1.setValueCount(3); - - UnionListWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - //set some values - writeListVector(writer2, new int[] {1, 2}); - writeListVector(writer2, new int[] {3, 4}); - writer2.setValueCount(3); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testListVectorEquals() { + public void testListVectorRangeEquals() { try (final ListVector vector1 = ListVector.empty("list", allocator); final ListVector vector2 = ListVector.empty("list", allocator);) { @@ -281,58 +148,28 @@ public void testListVectorEquals() { writeListVector(writer1, new int[] {1, 2}); writeListVector(writer1, new int[] {3, 4}); writeListVector(writer1, new int[] {5, 6}); - writer1.setValueCount(3); + writeListVector(writer1, new int[] {7, 8}); + writeListVector(writer1, new int[] {9, 10}); + writer1.setValueCount(5); UnionListWriter writer2 = vector2.getWriter(); writer2.allocate(); //set some values - writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {0, 0}); writeListVector(writer2, new int[] {3, 4}); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - writeListVector(writer2, new int[] {5, 6}); - writer2.setValueCount(3); + writeListVector(writer2, new int[] {7, 8}); + writeListVector(writer2, new int[] {0, 0}); + writer2.setValueCount(5); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 1, 1, 3); assertTrue(vector1.accept(visitor)); } } @Test - public void testStructVectorEqualsWithNull() { - - try (final StructVector vector1 = StructVector.empty("struct", allocator); - final StructVector vector2 = StructVector.empty("struct", allocator);) { - vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - - NullableStructWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - writeStructVector(writer1, 1, 10L); - writeStructVector(writer1, 2, 20L); - writeStructVector(writer1, 3, 30L); - writer1.setValueCount(3); - - NullableStructWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - writeStructVector(writer2, 1, 10L); - writeStructVector(writer2, 3, 30L); - writer2.setValueCount(3); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testStructVectorEquals() { + public void testStructVectorRangeEquals() { try (final StructVector vector1 = StructVector.empty("struct", allocator); final StructVector vector2 = StructVector.empty("struct", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -346,55 +183,27 @@ public void testStructVectorEquals() { writeStructVector(writer1, 1, 10L); writeStructVector(writer1, 2, 20L); writeStructVector(writer1, 3, 30L); - writer1.setValueCount(3); + writeStructVector(writer1, 4, 40L); + writeStructVector(writer1, 5, 50L); + writer1.setValueCount(5); NullableStructWriter writer2 = vector2.getWriter(); writer2.allocate(); - writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 0, 00L); writeStructVector(writer2, 2, 20L); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - writeStructVector(writer2, 3, 30L); - writer2.setValueCount(3); + writeStructVector(writer2, 4, 40L); + writeStructVector(writer2, 0, 0L); + writer2.setValueCount(5); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 1, 1, 3); assertTrue(vector1.accept(visitor)); } } @Test - public void testStructVectorEqualsWithDiffChild() { - try (final StructVector vector1 = StructVector.empty("struct", allocator); - final StructVector vector2 = StructVector.empty("struct", allocator);) { - vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); - vector2.addOrGet("f10", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); - - NullableStructWriter writer1 = vector1.getWriter(); - writer1.allocate(); - - writeStructVector(writer1, 1, 10L); - writeStructVector(writer1, 2, 20L); - writer1.setValueCount(2); - - NullableStructWriter writer2 = vector2.getWriter(); - writer2.allocate(); - - writeStructVector(writer2, 1, 10L); - writeStructVector(writer2, 2, 20L); - writer2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.accept(visitor)); - } - } - - @Test - public void testUnionVectorEquals() { + public void testUnionVectorRangeEquals() { try (final UnionVector vector1 = new UnionVector("union", allocator, null); final UnionVector vector2 = new UnionVector("union", allocator, null);) { @@ -411,37 +220,22 @@ public void testUnionVectorEquals() { vector1.setType(1, Types.MinorType.INT); vector1.setSafe(1, intHolder); - vector1.setValueCount(2); + + vector1.setType(2, Types.MinorType.INT); + vector1.setSafe(2, intHolder); + vector1.setValueCount(3); vector2.setType(0, Types.MinorType.UINT4); vector2.setSafe(0, uInt4Holder); vector2.setType(1, Types.MinorType.INT); vector2.setSafe(1, intHolder); - vector2.setValueCount(2); - - VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertTrue(vector1.accept(visitor)); - } - } - - @Test(expected = IllegalArgumentException.class) - public void testEqualsWithIndexOutOfRange() { - try (final IntVector vector1 = new IntVector("int", allocator); - final IntVector vector2 = new IntVector("int", allocator)) { - vector1.allocateNew(2); - vector1.setValueCount(2); - vector2.allocateNew(2); - vector2.setValueCount(2); - - vector1.setSafe(0, 1); - vector1.setSafe(1, 2); - - vector2.setSafe(0, 1); - vector2.setSafe(1, 2); + vector2.setType(2, Types.MinorType.INT); + vector2.setSafe(2, intHolder); + vector2.setValueCount(3); - RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 3, 2, 1); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector2, 1, 1, 2); assertTrue(vector1.accept(visitor)); } } @@ -451,12 +245,10 @@ public void testEqualsWithOutTypeCheck() { try (final IntVector intVector = new IntVector("int", allocator); final ZeroVector zeroVector = new ZeroVector()) { - VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector); - zeroVisitor.setTypeCheckNeeded(false); + VectorEqualsVisitor zeroVisitor = new VectorEqualsVisitor(zeroVector, false); assertTrue(intVector.accept(zeroVisitor)); - VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector); - intVisitor.setTypeCheckNeeded(false); + VectorEqualsVisitor intVisitor = new VectorEqualsVisitor(intVector, false); assertTrue(zeroVector.accept(intVisitor)); } }