From 6bc3f681cb0b32d46b2831168d3ad3fa293c1ff4 Mon Sep 17 00:00:00 2001 From: tianchen Date: Wed, 24 Jul 2019 18:34:52 +0800 Subject: [PATCH 01/13] ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal --- .../org/apache/arrow/vector/ValueVector.java | 25 +++ .../apache/arrow/vector/TestValueVector.java | 196 +++++++++++++++++- 2 files changed, 220 insertions(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 4d98397b29d..24669b604c4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -272,4 +272,29 @@ public interface ValueVector extends Closeable, Iterable { * @param from source vector */ void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); + + /** + * Check whether this vector equals to target vector. + * @param target target vector + * @return true if they have same type and elements, otherwise false + */ + default boolean equals(ValueVector target) { + if (this == target) { + return true; + } + if (target == null || this.getClass() != target.getClass()) { + return false; + } + + if (this.getValueCount() != target.getValueCount()) { + return false; + } + + for (int i = 0; i < getValueCount(); i++) { + if (!equals(i, target, i)) { + return false; + } + } + return true; + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index c38f4dc685b..2d94eaa36c2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -36,12 +36,21 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.NullableStructWriter; +import org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.arrow.vector.holders.NullableIntHolder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; import org.apache.arrow.vector.holders.NullableVarBinaryHolder; import org.apache.arrow.vector.holders.NullableVarCharHolder; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.Text; @@ -52,7 +61,6 @@ import io.netty.buffer.ArrowBuf; - public class TestValueVector { private static final String EMPTY_SCHEMA_PATH = ""; @@ -2245,4 +2253,190 @@ public void testGetNullFromVariableWidthVector() { assertNull(varBinaryVector.get(0)); } } + + @Test + public void testIntVectorEquals() { + try (final IntVector vector1 = new IntVector("v1", allocator); + final IntVector vector2 = new IntVector("v2", allocator)) { + + vector1.allocateNew(3); + vector1.setValueCount(3); + vector2.allocateNew(3); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + vector1.setSafe(2, 3); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + assertFalse(vector1.equals(vector2)); + + vector2.setValueCount(3); + vector2.setSafe(2, 2); + assertFalse(vector1.equals(vector2)); + + vector2.setSafe(2, 3); + assertTrue(vector1.equals(vector2)); + } + } + + @Test + public void testVarcharEquals() { + try (final VarCharVector vector1 = new VarCharVector("v1", allocator); + final VarCharVector vector2 = new VarCharVector("v2", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.equals(vector2)); + } + } + + @Test + public void testVarBinaryEquals() { + try (final VarBinaryVector vector1 = new VarBinaryVector("v1", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("v2", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setSafe(2, STR3, 0, STR3.length); + vector1.setValueCount(3); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setSafe(1, STR2, 0, STR2.length); + vector2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + + vector2.setSafe(2, STR3, 0, STR3.length); + vector2.setValueCount(3); + assertTrue(vector1.equals(vector2)); + } + } + + @Test + public void testListEquals() { + try (final ListVector vector1 = ListVector.empty("v1", allocator); + final ListVector vector2 = ListVector.empty("v2", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {5, 6}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + + writeListVector(writer2, new int[] {5, 6}); + writer2.setValueCount(3); + + assertTrue(vector1.equals(vector2)); + } + } + + @Test + public void testStructEquals() { + try (final StructVector vector1 = StructVector.empty("v1", allocator); + final StructVector vector2 = StructVector.empty("v2", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + assertTrue(vector1.equals(vector2)); + } + } + + @Test + public void testUnionEquals() { + try (final UnionVector vector1 = new UnionVector("v1", allocator, null); + final UnionVector vector2 = new UnionVector("v2", allocator, null);) { + + final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); + uInt4Holder.value = 10; + uInt4Holder.isSet = 1; + + final NullableIntHolder intHolder = new NullableIntHolder(); + uInt4Holder.value = 20; + uInt4Holder.isSet = 1; + + vector1.setType(0, Types.MinorType.UINT4); + vector1.setSafe(0, uInt4Holder); + + vector1.setType(1, Types.MinorType.INT); + vector1.setSafe(1, intHolder); + vector1.setValueCount(2); + + vector2.setType(0, Types.MinorType.UINT4); + vector2.setSafe(0, uInt4Holder); + + vector2.setType(1, Types.MinorType.INT); + vector2.setSafe(1, intHolder); + vector2.setValueCount(2); + + assertTrue(vector1.equals(vector2)); + } + } + + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { + writer.start(); + writer.integer("f0").writeInt(value1); + writer.bigInt("f1").writeBigInt(value2); + writer.end(); + } + + private void writeListVector(UnionListWriter writer, int[] values) { + writer.startList(); + for (int v: values) { + writer.integer().writeInt(v); + } + writer.endList(); + } } From 10dca2ccb6877b86795010b8b35dca9cff021bc5 Mon Sep 17 00:00:00 2001 From: tianchen Date: Wed, 24 Jul 2019 18:36:36 +0800 Subject: [PATCH 02/13] fix --- .../java/org/apache/arrow/vector/TestValueVector.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 2d94eaa36c2..2749c5eaff3 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -2283,7 +2283,7 @@ public void testIntVectorEquals() { } @Test - public void testVarcharEquals() { + public void testVarcharVectorEquals() { try (final VarCharVector vector1 = new VarCharVector("v1", allocator); final VarCharVector vector2 = new VarCharVector("v2", allocator)) { @@ -2308,7 +2308,7 @@ public void testVarcharEquals() { } @Test - public void testVarBinaryEquals() { + public void testVarBinaryVectorEquals() { try (final VarBinaryVector vector1 = new VarBinaryVector("v1", allocator); final VarBinaryVector vector2 = new VarBinaryVector("v2", allocator)) { @@ -2333,7 +2333,7 @@ public void testVarBinaryEquals() { } @Test - public void testListEquals() { + public void testListVectorEquals() { try (final ListVector vector1 = ListVector.empty("v1", allocator); final ListVector vector2 = ListVector.empty("v2", allocator);) { @@ -2363,7 +2363,7 @@ public void testListEquals() { } @Test - public void testStructEquals() { + public void testStructVectorEquals() { try (final StructVector vector1 = StructVector.empty("v1", allocator); final StructVector vector2 = StructVector.empty("v2", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -2395,7 +2395,7 @@ public void testStructEquals() { } @Test - public void testUnionEquals() { + public void testUnionVectorEquals() { try (final UnionVector vector1 = new UnionVector("v1", allocator, null); final UnionVector vector2 = new UnionVector("v2", allocator, null);) { From e58c15834afc426aa9a4ee941d4c1de99a414e6b Mon Sep 17 00:00:00 2001 From: tianchen Date: Thu, 25 Jul 2019 14:51:31 +0800 Subject: [PATCH 03/13] refactor Dictionary#equals --- .../main/codegen/templates/UnionVector.java | 7 +++--- .../arrow/vector/dictionary/Dictionary.java | 13 +--------- .../arrow/vector/TestDictionaryVector.java | 24 +++++++++---------- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index f288fb68c17..d4eee1667c5 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -672,9 +672,10 @@ public boolean equals(int index, ValueVector to, int toIndex) { ValueVector leftVector = getVector(index); ValueVector rightVector = that.getVector(toIndex); - if (leftVector.getClass() != rightVector.getClass()) { - return false; + if (leftVector == null) { + return rightVector == null; + } else { + return leftVector.equals(index, rightVector, toIndex); } - return leftVector.equals(index, rightVector, toIndex); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java index 082d2ba1744..df9eb6ace58 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java @@ -22,7 +22,6 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; -import org.apache.arrow.vector.util.Validator; /** * A dictionary (integer to Value mapping) that is used to facilitate @@ -64,21 +63,11 @@ public boolean equals(Object o) { return false; } Dictionary that = (Dictionary) o; - return Objects.equals(encoding, that.encoding) && compareFieldVector(dictionary, that.dictionary); + return Objects.equals(encoding, that.encoding) && dictionary.equals(that.dictionary); } @Override public int hashCode() { return Objects.hash(encoding, dictionary); } - - //TODO after vector api support compare two vectors, this should be cleaned up - private boolean compareFieldVector(FieldVector vector1, FieldVector vector2) { - try { - Validator.compareFieldVectors(vector1, vector2); - } catch (IllegalArgumentException e) { - return false; - } - return true; - } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 2d6391b33c1..83041cf1d34 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -398,8 +398,8 @@ public void testEncodeUnion() { @Test public void testIntEquals() { //test Int - try (final IntVector vector1 = new IntVector("", allocator); - final IntVector vector2 = new IntVector("", allocator)) { + try (final IntVector vector1 = new IntVector("v1", allocator); + final IntVector vector2 = new IntVector("v2", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -426,8 +426,8 @@ public void testIntEquals() { @Test public void testVarcharEquals() { - try (final VarCharVector vector1 = new VarCharVector("", allocator); - final VarCharVector vector2 = new VarCharVector("", allocator)) { + try (final VarCharVector vector1 = new VarCharVector("v1", allocator); + final VarCharVector vector2 = new VarCharVector("v2", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -455,8 +455,8 @@ public void testVarcharEquals() { @Test public void testVarBinaryEquals() { - try (final VarBinaryVector vector1 = new VarBinaryVector("", allocator); - final VarBinaryVector vector2 = new VarBinaryVector("", allocator)) { + try (final VarBinaryVector vector1 = new VarBinaryVector("v1", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("v2", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -484,8 +484,8 @@ public void testVarBinaryEquals() { @Test public void testListEquals() { - try (final ListVector vector1 = ListVector.empty("", allocator); - final ListVector vector2 = ListVector.empty("", allocator);) { + try (final ListVector vector1 = ListVector.empty("v1", allocator); + final ListVector vector2 = ListVector.empty("v2", allocator);) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -514,8 +514,8 @@ public void testListEquals() { @Test public void testStructEquals() { - try (final StructVector vector1 = StructVector.empty("", allocator); - final StructVector vector2 = StructVector.empty("", allocator);) { + try (final StructVector vector1 = StructVector.empty("v1", allocator); + final StructVector vector2 = StructVector.empty("v2", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -544,8 +544,8 @@ public void testStructEquals() { @Test public void testUnionEquals() { - try (final UnionVector vector1 = new UnionVector("", allocator, null); - final UnionVector vector2 = new UnionVector("", allocator, null);) { + try (final UnionVector vector1 = new UnionVector("v1", allocator, null); + final UnionVector vector2 = new UnionVector("v2", allocator, null);) { final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); uInt4Holder.value = 10; From 00268829690508bb211d308c88d07b660ac16759 Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 29 Jul 2019 15:36:45 +0800 Subject: [PATCH 04/13] use MinorType and check isSet --- java/vector/src/main/codegen/templates/UnionVector.java | 2 +- .../java/org/apache/arrow/vector/BaseFixedWidthVector.java | 6 +++++- .../org/apache/arrow/vector/BaseVariableWidthVector.java | 6 +++++- .../src/main/java/org/apache/arrow/vector/ValueVector.java | 2 +- .../apache/arrow/vector/complex/FixedSizeListVector.java | 2 +- .../java/org/apache/arrow/vector/complex/ListVector.java | 2 +- .../arrow/vector/complex/NonNullableStructVector.java | 2 +- 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index d4eee1667c5..cb6b7e40384 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -665,7 +665,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } UnionVector that = (UnionVector) to; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index aca34af5d85..7dde39dd344 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -870,12 +870,16 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } BaseFixedWidthVector that = (BaseFixedWidthVector) to; + if (this.isSet(index) != that.isSet(index)) { + return false; + } + int leftStart = typeWidth * index; int leftEnd = typeWidth * (index + 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 5bb54b91df9..6868e1f476b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -1369,12 +1369,16 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } BaseVariableWidthVector that = (BaseVariableWidthVector) to; + if (this.isSet(index) != that.isSet(index)) { + return false; + } + final int leftStart = getStartOffset(index); final int leftEnd = getStartOffset(index + 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 24669b604c4..4802ceae894 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -282,7 +282,7 @@ default boolean equals(ValueVector target) { if (this == target) { return true; } - if (target == null || this.getClass() != target.getClass()) { + if (target == null || this.getMinorType() != target.getMinorType()) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 39d0287210b..9485b91714a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -520,7 +520,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 43b43bdbe1a..530c0241ac2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -430,7 +430,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 1d9b8713697..48abcf3577e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -303,7 +303,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getClass() != to.getClass()) { + if (this.getMinorType() != to.getMinorType()) { return false; } NonNullableStructVector that = (NonNullableStructVector) to; From 3c9f066003ae914120c22cf75a93657e8e680a3d Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 29 Jul 2019 15:52:07 +0800 Subject: [PATCH 05/13] fix --- .../main/java/org/apache/arrow/vector/BaseFixedWidthVector.java | 2 +- .../java/org/apache/arrow/vector/BaseVariableWidthVector.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 7dde39dd344..906d7aaec84 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -876,7 +876,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { BaseFixedWidthVector that = (BaseFixedWidthVector) to; - if (this.isSet(index) != that.isSet(index)) { + if (this.isSet(index) != that.isSet(toIndex)) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 6868e1f476b..7b2deb2e174 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -1375,7 +1375,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { BaseVariableWidthVector that = (BaseVariableWidthVector) to; - if (this.isSet(index) != that.isSet(index)) { + if (this.isSet(index) != that.isSet(toIndex)) { return false; } From 1d95c9c7f98cff0c471c36a963c1b2817007593e Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 29 Jul 2019 20:47:58 +0800 Subject: [PATCH 06/13] fix Decimal equals --- .../apache/arrow/vector/DecimalVector.java | 31 +++++++++++++++++++ .../apache/arrow/vector/TestValueVector.java | 27 ++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index cf77186929c..1a6a2e8b8a2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -22,6 +22,7 @@ import java.math.BigDecimal; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.DecimalHolder; @@ -489,6 +490,36 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { set(index, isSet, start, buffer); } + @Override + public boolean equals(int index, ValueVector to, int toIndex) { + if (to == null) { + return false; + } + if (this.getMinorType() != to.getMinorType()) { + return false; + } + + DecimalVector that = (DecimalVector) to; + + if (this.scale != that.scale || this.precision != that.precision) { + return false; + } + + if (this.isSet(index) != that.isSet(toIndex)) { + return false; + } + + int leftStart = TYPE_WIDTH * index; + int leftEnd = TYPE_WIDTH * (index + 1); + + int rightStart = TYPE_WIDTH * toIndex; + int rightEnd = TYPE_WIDTH * (toIndex + 1); + + int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, + that.getDataBuffer(), rightStart, rightEnd); + return ret == 1; + } + /*----------------------------------------------------------------* | | diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 2749c5eaff3..3a52674d9dc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -2282,6 +2282,33 @@ public void testIntVectorEquals() { } } + @Test + public void testDecimalVectorEquals() { + try (final DecimalVector vector1 = new DecimalVector("v1", allocator, 3, 3); + final DecimalVector vector2 = new DecimalVector("v2", allocator, 3, 3); + final DecimalVector vector3 = new DecimalVector("v3", allocator, 3, 2)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + vector3.allocateNew(2); + vector3.setValueCount(2); + + vector1.setSafe(0, 100); + vector1.setSafe(1, 200); + + vector2.setSafe(0, 100); + vector2.setSafe(1, 200); + + vector3.setSafe(0, 100); + vector3.setSafe(1, 200); + + assertTrue(vector1.equals(vector2)); + assertFalse(vector1.equals(vector3)); + } + } + @Test public void testVarcharVectorEquals() { try (final VarCharVector vector1 = new VarCharVector("v1", allocator); From b942794b2b7b303f202f8f3d378d25d3ef290a81 Mon Sep 17 00:00:00 2001 From: tianchen Date: Wed, 31 Jul 2019 18:03:53 +0800 Subject: [PATCH 07/13] use ArrowType for equal --- .../main/codegen/templates/UnionVector.java | 2 +- .../arrow/vector/BaseFixedWidthVector.java | 2 +- .../arrow/vector/BaseVariableWidthVector.java | 2 +- .../apache/arrow/vector/DecimalVector.java | 32 ------------------- .../vector/complex/FixedSizeListVector.java | 2 +- .../arrow/vector/complex/ListVector.java | 2 +- .../complex/NonNullableStructVector.java | 2 +- 7 files changed, 6 insertions(+), 38 deletions(-) diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index cb6b7e40384..132534cd647 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -665,7 +665,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } UnionVector that = (UnionVector) to; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 906d7aaec84..ed521226a67 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -870,7 +870,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 7b2deb2e174..fd38a9b2e2a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -1369,7 +1369,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 1a6a2e8b8a2..5a450e6fe1b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -22,7 +22,6 @@ import java.math.BigDecimal; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.DecimalHolder; @@ -490,37 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) { set(index, isSet, start, buffer); } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - if (this.getMinorType() != to.getMinorType()) { - return false; - } - - DecimalVector that = (DecimalVector) to; - - if (this.scale != that.scale || this.precision != that.precision) { - return false; - } - - if (this.isSet(index) != that.isSet(toIndex)) { - return false; - } - - int leftStart = TYPE_WIDTH * index; - int leftEnd = TYPE_WIDTH * (index + 1); - - int rightStart = TYPE_WIDTH * toIndex; - int rightEnd = TYPE_WIDTH * (toIndex + 1); - - int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, - that.getDataBuffer(), rightStart, rightEnd); - return ret == 1; - } - - /*----------------------------------------------------------------* | | | vector transfer | diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 9485b91714a..6f20ebf0767 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -520,7 +520,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 530c0241ac2..813cd4b3a5b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -430,7 +430,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 48abcf3577e..1126e2a771e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -303,7 +303,7 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (this.getMinorType() != to.getMinorType()) { + if (!this.getField().getType().equals(to.getField().getType())) { return false; } NonNullableStructVector that = (NonNullableStructVector) to; From c7081c2746bb856fabcd87cb9069a2fc42d95da9 Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 5 Aug 2019 15:36:17 +0800 Subject: [PATCH 08/13] check list validity bit --- .../org/apache/arrow/vector/complex/FixedSizeListVector.java | 4 ++++ .../java/org/apache/arrow/vector/complex/ListVector.java | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 6f20ebf0767..e27241eeac6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -526,6 +526,10 @@ public boolean equals(int index, ValueVector to, int toIndex) { FixedSizeListVector that = (FixedSizeListVector) to; + if (this.isSet(index) != that.isSet(toIndex)) { + return false; + } + for (int i = 0; i < listSize; i++) { if (!vector.equals(index * listSize + i, that, toIndex * listSize + i)) { return false; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 813cd4b3a5b..d5bf818c711 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -435,6 +435,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { } ListVector that = (ListVector) to; + + if (this.isSet(index) != that.isSet(toIndex)) { + return false; + } + final int leftStart = offsetBuffer.getInt(index * OFFSET_WIDTH); final int leftEnd = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); From 0dfa943e95cb0238e1562a485f0a511a2a43ba1a Mon Sep 17 00:00:00 2001 From: tianchen Date: Mon, 5 Aug 2019 16:27:56 +0800 Subject: [PATCH 09/13] compare struct child names and add UT --- .../org/apache/arrow/vector/ValueVector.java | 2 +- .../complex/NonNullableStructVector.java | 5 + .../arrow/vector/complex/StructVector.java | 45 +++++++ .../apache/arrow/vector/TestValueVector.java | 119 ++++++++++++++++++ 4 files changed, 170 insertions(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 4802ceae894..485056d76b5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -282,7 +282,7 @@ default boolean equals(ValueVector target) { if (this == target) { return true; } - if (target == null || this.getMinorType() != target.getMinorType()) { + if (target == null || !this.getField().getType().equals(target.getField().getType())) { return false; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 1126e2a771e..8ffb6dab674 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -307,9 +307,14 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } NonNullableStructVector that = (NonNullableStructVector) to; + List leftChildrens = new ArrayList<>(); List rightChildrens = new ArrayList<>(); + if (!getChildFieldNames().equals(that.getChildFieldNames())) { + return false; + } + for (String child : getChildFieldNames()) { ValueVector v = getChild(child); if (v != null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index 2ad9fb75091..f39afb53bb7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -488,6 +488,51 @@ public int hashCode(int index) { } } + @Override + public boolean equals(int index, ValueVector to, int toIndex) { + if (to == null) { + return false; + } + if (!this.getField().getType().equals(to.getField().getType())) { + return false; + } + StructVector that = (StructVector) to; + if (this.isSet(index) != that.isSet(toIndex)) { + return false; + } + List leftChildrens = new ArrayList<>(); + List rightChildrens = new ArrayList<>(); + + if (!getChildFieldNames().equals(that.getChildFieldNames())) { + return false; + } + + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null) { + leftChildrens.add(v); + } + } + + for (String child : that.getChildFieldNames()) { + ValueVector v = that.getChild(child); + if (v != null) { + rightChildrens.add(v); + } + } + + if (leftChildrens.size() != rightChildrens.size()) { + return false; + } + + for (int i = 0; i < leftChildrens.size(); i++) { + if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) { + return false; + } + } + return true; + } + @Override public void get(int index, ComplexHolder holder) { holder.isSet = isSet(index); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 3a52674d9dc..876c747cb04 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -2254,6 +2254,25 @@ public void testGetNullFromVariableWidthVector() { } } + @Test + public void testIntVectorEqualsWithNull() { + try (final IntVector vector1 = new IntVector("v1", allocator); + final IntVector vector2 = new IntVector("v2", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + + assertFalse(vector1.equals(vector2)); + } + } + @Test public void testIntVectorEquals() { try (final IntVector vector1 = new IntVector("v1", allocator); @@ -2309,6 +2328,25 @@ public void testDecimalVectorEquals() { } } + @Test + public void testVarcharVectorEuqalsWithNull() { + try (final VarCharVector vector1 = new VarCharVector("v1", allocator); + final VarCharVector vector2 = new VarCharVector("v2", allocator)) { + + vector1.allocateNew(); + vector2.allocateNew(); + + // set some values + vector1.setSafe(0, STR1, 0, STR1.length); + vector1.setSafe(1, STR2, 0, STR2.length); + vector1.setValueCount(2); + + vector2.setSafe(0, STR1, 0, STR1.length); + vector2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + } + } + @Test public void testVarcharVectorEquals() { try (final VarCharVector vector1 = new VarCharVector("v1", allocator); @@ -2359,6 +2397,32 @@ public void testVarBinaryVectorEquals() { } } + @Test + public void testListVectorEqualsWithNull() { + try (final ListVector vector1 = ListVector.empty("v1", allocator); + final ListVector vector2 = ListVector.empty("v2", allocator);) { + + UnionListWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + //set some values + writeListVector(writer1, new int[] {1, 2}); + writeListVector(writer1, new int[] {3, 4}); + writeListVector(writer1, new int[] {}); + writer1.setValueCount(3); + + UnionListWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + //set some values + writeListVector(writer2, new int[] {1, 2}); + writeListVector(writer2, new int[] {3, 4}); + writer2.setValueCount(3); + + assertFalse(vector1.equals(vector2)); + } + } + @Test public void testListVectorEquals() { try (final ListVector vector1 = ListVector.empty("v1", allocator); @@ -2389,6 +2453,35 @@ public void testListVectorEquals() { } } + @Test + public void testStructVectorEqualsWithNull() { + + try (final StructVector vector1 = StructVector.empty("v1", allocator); + final StructVector vector2 = StructVector.empty("v2", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writeStructVector(writer1, 3, 30L); + writer1.setValueCount(3); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 3, 30L); + writer2.setValueCount(3); + + assertFalse(vector1.equals(vector2)); + } + } + @Test public void testStructVectorEquals() { try (final StructVector vector1 = StructVector.empty("v1", allocator); @@ -2421,6 +2514,32 @@ public void testStructVectorEquals() { } } + @Test + public void testStructVectorEqualsWithDiffChild() { + try (final StructVector vector1 = StructVector.empty("v1", allocator); + final StructVector vector2 = StructVector.empty("v2", allocator);) { + vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + vector2.addOrGet("f10", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); + + NullableStructWriter writer1 = vector1.getWriter(); + writer1.allocate(); + + writeStructVector(writer1, 1, 10L); + writeStructVector(writer1, 2, 20L); + writer1.setValueCount(2); + + NullableStructWriter writer2 = vector2.getWriter(); + writer2.allocate(); + + writeStructVector(writer2, 1, 10L); + writeStructVector(writer2, 2, 20L); + writer2.setValueCount(2); + assertFalse(vector1.equals(vector2)); + } + } + @Test public void testUnionVectorEquals() { try (final UnionVector vector1 = new UnionVector("v1", allocator, null); From 694d9f6b96538ce8011a77b4c4a85df0482aa56d Mon Sep 17 00:00:00 2001 From: tianchen Date: Thu, 8 Aug 2019 23:04:46 +0800 Subject: [PATCH 10/13] make equals to visitor mode --- .../main/codegen/templates/UnionVector.java | 7 + .../arrow/vector/BaseFixedWidthVector.java | 35 +- .../arrow/vector/BaseVariableWidthVector.java | 27 +- .../org/apache/arrow/vector/ValueVector.java | 27 +- .../org/apache/arrow/vector/ZeroVector.java | 6 + .../vector/compare/RangeEqualsVisitor.java | 300 ++++++++++++++++++ .../vector/compare/VectorEqualsVisitor.java | 84 +++++ .../vector/complex/AbstractStructVector.java | 5 +- .../vector/complex/FixedSizeListVector.java | 17 +- .../arrow/vector/complex/ListVector.java | 32 +- .../complex/NonNullableStructVector.java | 6 + .../arrow/vector/complex/StructVector.java | 48 +-- .../arrow/vector/dictionary/Dictionary.java | 3 +- .../apache/arrow/vector/TestValueVector.java | 59 ++-- .../vector/types/pojo/TestExtensionType.java | 6 + 15 files changed, 567 insertions(+), 95 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 132534cd647..2d3c8d95f5f 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -20,6 +20,7 @@ import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.types.UnionMode; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; @@ -36,6 +37,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -678,4 +680,9 @@ public boolean equals(int index, ValueVector to, int toIndex) { return leftVector.equals(index, rightVector, toIndex); } } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index ed521226a67..13898bead1a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; @@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final } + public int getTypeWidth() { + return typeWidth; + } + @Override public String getName() { return field.getName(); @@ -876,18 +881,32 @@ public boolean equals(int index, ValueVector to, int toIndex) { BaseFixedWidthVector that = (BaseFixedWidthVector) to; - if (this.isSet(index) != that.isSet(toIndex)) { + boolean isNull = isNull(index); + if (isNull != that.isNull(toIndex)) { return false; } - int leftStart = typeWidth * index; - int leftEnd = typeWidth * (index + 1); + if (!isNull) { + if (this.isSet(index) != that.isSet(toIndex)) { + return false; + } - int rightStart = typeWidth * toIndex; - int rightEnd = typeWidth * (toIndex + 1); + int leftStart = typeWidth * index; + int leftEnd = typeWidth * (index + 1); - int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, - that.getDataBuffer(), rightStart, rightEnd); - return ret == 1; + int rightStart = typeWidth * toIndex; + int rightEnd = typeWidth * (toIndex + 1); + + int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, + that.getDataBuffer(), rightStart, rightEnd); + return ret == 1; + } + + return true; + } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index fd38a9b2e2a..9895120f858 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -28,6 +28,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.CallBack; @@ -1375,18 +1376,28 @@ public boolean equals(int index, ValueVector to, int toIndex) { BaseVariableWidthVector that = (BaseVariableWidthVector) to; - if (this.isSet(index) != that.isSet(toIndex)) { + boolean isNull = isNull(index); + if (isNull != that.isNull(toIndex)) { return false; } - final int leftStart = getStartOffset(index); - final int leftEnd = getStartOffset(index + 1); + if (!isNull) { + final int leftStart = getStartOffset(index); + final int leftEnd = getStartOffset(index + 1); - final int rightStart = that.getStartOffset(toIndex); - final int rightEnd = that.getStartOffset(toIndex + 1); + final int rightStart = that.getStartOffset(toIndex); + final int rightEnd = that.getStartOffset(toIndex + 1); - int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, - that.getDataBuffer(), rightStart, rightEnd); - return ret == 1; + int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, + that.getDataBuffer(), rightStart, rightEnd); + return ret == 1; + } + + return true; + } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 485056d76b5..aed24f259ee 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -21,6 +21,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.Field; @@ -274,27 +275,9 @@ public interface ValueVector extends Closeable, Iterable { void copyFromSafe(int fromIndex, int thisIndex, ValueVector from); /** - * Check whether this vector equals to target vector. - * @param target target vector - * @return true if they have same type and elements, otherwise false + * Compare range values in this vector and vector in visitor. + * @param visitor visitor which holds the vector to compare. + * @return true if equals, otherwise false. */ - default boolean equals(ValueVector target) { - if (this == target) { - return true; - } - if (target == null || !this.getField().getType().equals(target.getField().getType())) { - return false; - } - - if (this.getValueCount() != target.getValueCount()) { - return false; - } - - for (int i = 0; i < getValueCount(); i++) { - if (!equals(i, target, i)) { - return false; - } - } - return true; - } + boolean accept(RangeEqualsVisitor visitor); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 2a6e21c8905..69174329e25 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -25,6 +25,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.NullReader; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return false; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java new file mode 100644 index 00000000000..94640ef3cba --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compare; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.arrow.memory.util.ByteFunctionHelpers; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.UnionVector; + +/** + * Visitor to compare a range of values for vectors. + */ +public class RangeEqualsVisitor { + + protected final ValueVector right; + protected final int leftStart; + protected final int rightStart; + protected final int length; + + /** + * Constructs a new instance. + */ + public RangeEqualsVisitor(ValueVector right, int leftStart, int rightStart, int length) { + this.leftStart = leftStart; + this.rightStart = rightStart; + this.right = right; + this.length = length; + } + + public boolean visit(BaseFixedWidthVector left) { + return compareBaseFixedWidthVectors(left); + } + + public boolean visit(BaseVariableWidthVector left) { + return compareBaseVariableWidthVectors(left); + } + + public boolean visit(ListVector left) { + return compareListVectors(left); + } + + public boolean visit(FixedSizeListVector left) { + return compareFixedSizeListVectors(left); + } + + public boolean visit(NonNullableStructVector left) { + return compareStructVectors(left); + } + + public boolean visit(UnionVector left) { + return compareUnionVectors(left); + } + + public boolean visit(ZeroVector left) { + return false; + } + + protected boolean compareUnionVectors(UnionVector left) { + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + UnionVector rightVector = (UnionVector) right; + + List leftChildrens = left.getChildrenFromFields(); + List rightChildrens = rightVector.getChildrenFromFields(); + + if (leftChildrens.size() != rightChildrens.size()) { + return false; + } + + for (int k = 0; k < leftChildrens.size(); k++) { + RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightChildrens.get(k), + leftStart, rightStart, length); + if (!leftChildrens.get(k).accept(visitor)) { + return false; + } + } + return true; + } + + protected boolean compareStructVectors(NonNullableStructVector left) { + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + NonNullableStructVector rightVector = (NonNullableStructVector) right; + + if (!left.getChildFieldNames().equals(rightVector.getChildFieldNames())) { + return false; + } + + List leftChildrens = new ArrayList<>(); + List rightChildrens = new ArrayList<>(); + + for (String child : left.getChildFieldNames()) { + ValueVector v = left.getChild(child); + if (v != null) { + leftChildrens.add(v); + } + } + + for (String child : rightVector.getChildFieldNames()) { + ValueVector v = rightVector.getChild(child); + if (v != null) { + rightChildrens.add(v); + } + } + + if (leftChildrens.size() != rightChildrens.size()) { + return false; + } + + for (int k = 0; k < leftChildrens.size(); k++) { + RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightChildrens.get(k), + leftStart, rightStart, length); + if (!leftChildrens.get(k).accept(visitor)) { + return false; + } + } + return true; + } + + protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { + + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + int typeWidth = left.getTypeWidth(); + + for (int i = 0; i < length; i++) { + int leftIndex = leftStart + i; + int rightIndex = rightStart + i; + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + if (!isNull) { + int start1 = typeWidth * leftIndex; + int end1 = typeWidth * (leftIndex + 1); + + int start2 = typeWidth * rightIndex; + int end2 = typeWidth * (rightIndex + 1); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, + right.getDataBuffer(), start2, end2); + + if (ret == 0) { + return false; + } + + } + } + + return true; + } + + protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) { + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + for (int i = 0; i < length; i++) { + int leftIndex = leftStart + i; + int rightIndex = rightStart + i; + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; + + if (!isNull) { + final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, + right.getDataBuffer(), start2, end2); + + if (ret == 0) { + return false; + } + } + } + return true; + } + + protected boolean compareListVectors(ListVector left) { + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + for (int i = 0; i < length; i++) { + int leftIndex = leftStart + i; + int rightIndex = rightStart + i; + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; + + if (!isNull) { + final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + if ((end1 - start1) != (end2 - start2)) { + return false; + } + + ValueVector dataVector1 = left.getDataVector(); + ValueVector dataVector2 = ((ListVector)right).getDataVector(); + + if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { + return false; + } + } + } + return true; + } + + protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { + if (!left.getField().getType().equals(right.getField().getType())) { + return false; + } + + if (left.getListSize() != ((FixedSizeListVector)right).getListSize()) { + return false; + } + + for (int i = 0; i < length; i++) { + int leftIndex = leftStart + i; + int rightIndex = rightStart + i; + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int listSize = left.getListSize(); + + if (!isNull) { + final int start1 = leftIndex * listSize; + final int end1 = (leftIndex + 1) * listSize; + + final int start2 = rightIndex * listSize; + final int end2 = (rightIndex + 1) * listSize; + + if ((end1 - start1) != (end2 - start2)) { + return false; + } + + ValueVector dataVector1 = left.getDataVector(); + ValueVector dataVector2 = ((FixedSizeListVector)right).getDataVector(); + + if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { + return false; + } + } + } + return true; + } + +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java new file mode 100644 index 00000000000..9b3c240ed78 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compare; + +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.UnionVector; + +/** + * Visitor to compare vectors equal. + */ +public class VectorEqualsVisitor extends RangeEqualsVisitor { + + public VectorEqualsVisitor(ValueVector right) { + super(right, 0, 0, right.getValueCount()); + } + + @Override + protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareBaseFixedWidthVectors(left); + } + + @Override + protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareBaseVariableWidthVectors(left); + } + + @Override + protected boolean compareListVectors(ListVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareListVectors(left); + } + + @Override + protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareFixedSizeListVectors(left); + } + + @Override + protected boolean compareStructVectors(NonNullableStructVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareStructVectors(left); + } + + @Override + protected boolean compareUnionVectors(UnionVector left) { + if (left.getValueCount() != right.getValueCount()) { + return false; + } + return super.compareUnionVectors(left); + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java index dc9b1a1ab2e..25762fda88c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java @@ -226,7 +226,10 @@ protected List getChildren() { return children; } - protected List getChildFieldNames() { + /** + * Get child field names. + */ + public List getChildFieldNames() { return getChildren().stream() .map(child -> child.getField().getName()) .collect(Collectors.toList()); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index e27241eeac6..8d09aa43f56 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -39,6 +39,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; @@ -526,18 +527,26 @@ public boolean equals(int index, ValueVector to, int toIndex) { FixedSizeListVector that = (FixedSizeListVector) to; - if (this.isSet(index) != that.isSet(toIndex)) { + boolean isNull = isNull(index); + if (isNull != that.isNull(toIndex)) { return false; } - for (int i = 0; i < listSize; i++) { - if (!vector.equals(index * listSize + i, that, toIndex * listSize + i)) { - return false; + if (!isNull) { + for (int i = 0; i < listSize; i++) { + if (!vector.equals(index * listSize + i, that, toIndex * listSize + i)) { + return false; + } } } return true; } + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); + } + private class TransferImpl implements TransferPair { FixedSizeListVector to; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index d5bf818c711..29c8e43f4fa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -34,6 +34,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.complex.impl.UnionListWriter; @@ -436,26 +437,28 @@ public boolean equals(int index, ValueVector to, int toIndex) { ListVector that = (ListVector) to; - if (this.isSet(index) != that.isSet(toIndex)) { + boolean isNull = isNull(index); + if (isNull != that.isNull(toIndex)) { return false; } - final int leftStart = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int leftEnd = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); + if (!isNull) { + final int leftStart = offsetBuffer.getInt(index * OFFSET_WIDTH); + final int leftEnd = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); - final int rightStart = that.offsetBuffer.getInt(toIndex * OFFSET_WIDTH); - final int rightEnd = that.offsetBuffer.getInt((toIndex + 1) * OFFSET_WIDTH); + final int rightStart = that.offsetBuffer.getInt(toIndex * OFFSET_WIDTH); + final int rightEnd = that.offsetBuffer.getInt((toIndex + 1) * OFFSET_WIDTH); - if ((leftEnd - leftStart) != (rightEnd - rightStart)) { - return false; - } - - for (int i = 0; i < (leftEnd - leftStart); i++) { - if (!vector.equals(leftStart + i, that.vector, rightStart + i)) { + if ((leftEnd - leftStart) != (rightEnd - rightStart)) { return false; } - } + for (int i = 0; i < (leftEnd - leftStart); i++) { + if (!vector.equals(leftStart + i, that.vector, rightStart + i)) { + return false; + } + } + } return true; } @@ -842,4 +845,9 @@ public void setLastSet(int value) { public int getLastSet() { return lastSet; } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 8ffb6dab674..9b457574368 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -29,6 +29,7 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.holders.ComplexHolder; @@ -298,6 +299,11 @@ public int hashCode(int index) { return hash; } + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(this); + } + @Override public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index f39afb53bb7..a7da368bee4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -497,39 +497,45 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } StructVector that = (StructVector) to; - if (this.isSet(index) != that.isSet(toIndex)) { - return false; - } - List leftChildrens = new ArrayList<>(); - List rightChildrens = new ArrayList<>(); - if (!getChildFieldNames().equals(that.getChildFieldNames())) { + boolean isNull = isNull(index); + if (isNull != that.isNull(toIndex)) { return false; } - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null) { - leftChildrens.add(v); + if (!isNull) { + List leftChildrens = new ArrayList<>(); + List rightChildrens = new ArrayList<>(); + + if (!getChildFieldNames().equals(that.getChildFieldNames())) { + return false; } - } - for (String child : that.getChildFieldNames()) { - ValueVector v = that.getChild(child); - if (v != null) { - rightChildrens.add(v); + for (String child : getChildFieldNames()) { + ValueVector v = getChild(child); + if (v != null) { + leftChildrens.add(v); + } } - } - if (leftChildrens.size() != rightChildrens.size()) { - return false; - } + for (String child : that.getChildFieldNames()) { + ValueVector v = that.getChild(child); + if (v != null) { + rightChildrens.add(v); + } + } - for (int i = 0; i < leftChildrens.size(); i++) { - if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) { + if (leftChildrens.size() != rightChildrens.size()) { return false; } + + for (int i = 0; i < leftChildrens.size(); i++) { + if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) { + return false; + } + } } + return true; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java index df9eb6ace58..72b3da0967b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java @@ -20,6 +20,7 @@ import java.util.Objects; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.compare.VectorEqualsVisitor; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; @@ -63,7 +64,7 @@ public boolean equals(Object o) { return false; } Dictionary that = (Dictionary) o; - return Objects.equals(encoding, that.encoding) && dictionary.equals(that.dictionary); + return Objects.equals(encoding, that.encoding) && dictionary.accept(new VectorEqualsVisitor(that.dictionary)); } @Override diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 876c747cb04..fc5c6d42790 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -36,6 +36,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; +import org.apache.arrow.vector.compare.VectorEqualsVisitor; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; @@ -2268,8 +2269,9 @@ public void testIntVectorEqualsWithNull() { vector1.setSafe(1, 2); vector2.setSafe(0, 1); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); - assertFalse(vector1.equals(vector2)); + assertFalse(vector1.accept(visitor)); } } @@ -2290,14 +2292,16 @@ public void testIntVectorEquals() { vector2.setSafe(0, 1); vector2.setSafe(1, 2); - assertFalse(vector1.equals(vector2)); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); vector2.setValueCount(3); vector2.setSafe(2, 2); assertFalse(vector1.equals(vector2)); vector2.setSafe(2, 3); - assertTrue(vector1.equals(vector2)); + assertTrue(vector1.accept(visitor)); } } @@ -2323,8 +2327,11 @@ public void testDecimalVectorEquals() { vector3.setSafe(0, 100); vector3.setSafe(1, 200); - assertTrue(vector1.equals(vector2)); - assertFalse(vector1.equals(vector3)); + VectorEqualsVisitor visitor1 = new VectorEqualsVisitor(vector2); + VectorEqualsVisitor visitor2 = new VectorEqualsVisitor(vector3); + + assertTrue(vector1.accept(visitor1)); + assertFalse(vector1.accept(visitor2)); } } @@ -2343,7 +2350,9 @@ public void testVarcharVectorEuqalsWithNull() { vector2.setSafe(0, STR1, 0, STR1.length); vector2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); } } @@ -2364,11 +2373,13 @@ public void testVarcharVectorEquals() { vector2.setSafe(0, STR1, 0, STR1.length); vector2.setSafe(1, STR2, 0, STR2.length); vector2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); vector2.setSafe(2, STR3, 0, STR3.length); vector2.setValueCount(3); - assertTrue(vector1.equals(vector2)); + assertTrue(vector1.accept(visitor)); } } @@ -2389,11 +2400,13 @@ public void testVarBinaryVectorEquals() { vector2.setSafe(0, STR1, 0, STR1.length); vector2.setSafe(1, STR2, 0, STR2.length); vector2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); vector2.setSafe(2, STR3, 0, STR3.length); vector2.setValueCount(3); - assertTrue(vector1.equals(vector2)); + assertTrue(vector1.accept(visitor)); } } @@ -2419,7 +2432,9 @@ public void testListVectorEqualsWithNull() { writeListVector(writer2, new int[] {3, 4}); writer2.setValueCount(3); - assertFalse(vector1.equals(vector2)); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + + assertFalse(vector1.accept(visitor)); } } @@ -2444,12 +2459,14 @@ public void testListVectorEquals() { writeListVector(writer2, new int[] {1, 2}); writeListVector(writer2, new int[] {3, 4}); writer2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); writeListVector(writer2, new int[] {5, 6}); writer2.setValueCount(3); - assertTrue(vector1.equals(vector2)); + assertTrue(vector1.accept(visitor)); } } @@ -2478,7 +2495,8 @@ public void testStructVectorEqualsWithNull() { writeStructVector(writer2, 3, 30L); writer2.setValueCount(3); - assertFalse(vector1.equals(vector2)); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); } } @@ -2505,12 +2523,14 @@ public void testStructVectorEquals() { writeStructVector(writer2, 1, 10L); writeStructVector(writer2, 2, 20L); writer2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); writeStructVector(writer2, 3, 30L); writer2.setValueCount(3); - assertTrue(vector1.equals(vector2)); + assertTrue(vector1.accept(visitor)); } } @@ -2536,7 +2556,9 @@ public void testStructVectorEqualsWithDiffChild() { writeStructVector(writer2, 1, 10L); writeStructVector(writer2, 2, 20L); writer2.setValueCount(2); - assertFalse(vector1.equals(vector2)); + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertFalse(vector1.accept(visitor)); } } @@ -2567,7 +2589,8 @@ public void testUnionVectorEquals() { vector2.setSafe(1, intHolder); vector2.setValueCount(2); - assertTrue(vector1.equals(vector2)); + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 792bd29903b..21530985650 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -36,6 +36,7 @@ import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; @@ -215,6 +216,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { return getUnderlyingVector().equals(index, to, toIndex); } + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(getUnderlyingVector()); + } + public void set(int index, UUID uuid) { ByteBuffer bb = ByteBuffer.allocate(16); bb.putLong(uuid.getMostSignificantBits()); From 226a20f8c96e04ac89a0ba0e7ca833f5f4edd404 Mon Sep 17 00:00:00 2001 From: tianchen Date: Fri, 9 Aug 2019 16:34:15 +0800 Subject: [PATCH 11/13] refactor --- .../main/codegen/templates/UnionVector.java | 3 + .../arrow/vector/BaseFixedWidthVector.java | 27 +-- .../arrow/vector/BaseVariableWidthVector.java | 22 +- .../arrow/vector/ExtensionTypeVector.java | 6 + .../org/apache/arrow/vector/ZeroVector.java | 2 +- .../arrow/vector/compare/CompareUtility.java | 226 ++++++++++++++++++ .../vector/compare/RangeEqualsVisitor.java | 155 +++--------- .../vector/compare/VectorEqualsVisitor.java | 54 +---- .../vector/complex/FixedSizeListVector.java | 17 +- .../arrow/vector/complex/ListVector.java | 27 +-- .../complex/NonNullableStructVector.java | 35 +-- .../arrow/vector/complex/StructVector.java | 51 ---- .../arrow/vector/TestDictionaryVector.java | 24 +- .../apache/arrow/vector/TestValueVector.java | 80 +++++-- .../vector/types/pojo/TestExtensionType.java | 6 - 15 files changed, 358 insertions(+), 377 deletions(-) create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 2d3c8d95f5f..2880d9eb32f 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; @@ -670,6 +671,8 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (!this.getField().getType().equals(to.getField().getType())) { return false; } + CompareUtility.checkIndices(this, index, to, toIndex); + UnionVector that = (UnionVector) to; ValueVector leftVector = getVector(index); ValueVector rightVector = that.getVector(toIndex); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 13898bead1a..8ba32b848cc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; @@ -875,34 +876,16 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } + if (!this.getField().getType().equals(to.getField().getType())) { return false; } - BaseFixedWidthVector that = (BaseFixedWidthVector) to; - - boolean isNull = isNull(index); - if (isNull != that.isNull(toIndex)) { - return false; - } + CompareUtility.checkIndices(this, index, to, toIndex); - if (!isNull) { - if (this.isSet(index) != that.isSet(toIndex)) { - return false; - } - - int leftStart = typeWidth * index; - int leftEnd = typeWidth * (index + 1); - - int rightStart = typeWidth * toIndex; - int rightEnd = typeWidth * (toIndex + 1); - - int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, - that.getDataBuffer(), rightStart, rightEnd); - return ret == 1; - } + BaseFixedWidthVector that = (BaseFixedWidthVector) to; - return true; + return CompareUtility.compare(this, index, that, toIndex); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 9895120f858..67edde4e68c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -28,6 +28,7 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; @@ -1374,26 +1375,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } - BaseVariableWidthVector that = (BaseVariableWidthVector) to; - - boolean isNull = isNull(index); - if (isNull != that.isNull(toIndex)) { - return false; - } + CompareUtility.checkIndices(this, index, to, toIndex); - if (!isNull) { - final int leftStart = getStartOffset(index); - final int leftEnd = getStartOffset(index + 1); - - final int rightStart = that.getStartOffset(toIndex); - final int rightEnd = that.getStartOffset(toIndex + 1); - - int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd, - that.getDataBuffer(), rightStart, rightEnd); - return ret == 1; - } + BaseVariableWidthVector that = (BaseVariableWidthVector) to; - return true; + return CompareUtility.compare(this, index, that, toIndex); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java index 14a66f8dafa..02dc8e3f438 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java @@ -22,6 +22,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.reader.FieldReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.Types.MinorType; @@ -256,4 +257,9 @@ public Iterator iterator() { public BufferAllocator getAllocator() { return underlyingVector.getAllocator(); } + + @Override + public boolean accept(RangeEqualsVisitor visitor) { + return visitor.visit(getUnderlyingVector()); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 69174329e25..4d5826c8935 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -268,6 +268,6 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { @Override public boolean accept(RangeEqualsVisitor visitor) { - return false; + return true; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java new file mode 100644 index 00000000000..8bab136c8f1 --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compare; + +import org.apache.arrow.memory.util.ByteFunctionHelpers; +import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.NonNullableStructVector; + +/** + * Utility to compare two single values in two vectors with same type. + */ +public class CompareUtility { + + /** + * Compare {@link BaseFixedWidthVector} value equals with given indices and vectors. + * @param left left vector to compare + * @param leftIndex left index + * @param right right vector to compare + * @param rightIndex right index + * @return true if equals, otherwise false + */ + public static boolean compare(BaseFixedWidthVector left, int leftIndex, BaseFixedWidthVector right, int rightIndex) { + + boolean isNull = left.isNull(leftIndex); + + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int typeWidth = left.getTypeWidth(); + if (!isNull) { + int startByteLeft = typeWidth * leftIndex; + int endByteLeft = typeWidth * (leftIndex + 1); + + int startByteRight = typeWidth * rightIndex; + int endByteRight = typeWidth * (rightIndex + 1); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, + right.getDataBuffer(), startByteRight, endByteRight); + + if (ret == 0) { + return false; + } + + } + return true; + } + + /** + * Compare {@link BaseVariableWidthVector} value equals with given indices and vectors. + * @param left left vector to compare + * @param leftIndex left index + * @param right right vector to compare + * @param rightIndex right index + * @return true if equals, otherwise false + */ + public static boolean compare(BaseVariableWidthVector left, int leftIndex, + BaseVariableWidthVector right, int rightIndex) { + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; + + if (!isNull) { + final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, + right.getDataBuffer(), start2, end2); + + if (ret == 0) { + return false; + } + } + return true; + } + + /** + * Compare {@link ListVector} value equals with given indices and vectors. + * @param left left vector to compare + * @param leftIndex left index + * @param right right vector to compare + * @param rightIndex right index + * @return true if equals, otherwise false + */ + public static boolean compare(ListVector left, int leftIndex, ListVector right, int rightIndex) { + + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; + + if (!isNull) { + final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { + return false; + } + + ValueVector dataVector1 = left.getDataVector(); + ValueVector dataVector2 = right.getDataVector(); + + if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, startByteLeft, + startByteRight, (endByteLeft - startByteLeft)))) { + return false; + } + } + return true; + } + + /** + * Compare {@link FixedSizeListVector} value equals with given indices and vectors. + * @param left left vector to compare + * @param leftIndex left index + * @param right right vector to compare + * @param rightIndex right index + * @return true if equals, otherwise false + */ + public static boolean compare(FixedSizeListVector left, int leftIndex, FixedSizeListVector right, int rightIndex) { + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + int listSize = left.getListSize(); + + if (!isNull) { + final int start1 = leftIndex * listSize; + final int end1 = (leftIndex + 1) * listSize; + + final int start2 = rightIndex * listSize; + final int end2 = (rightIndex + 1) * listSize; + + if ((end1 - start1) != (end2 - start2)) { + return false; + } + + ValueVector dataVector1 = left.getDataVector(); + ValueVector dataVector2 = right.getDataVector(); + + if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { + return false; + } + } + return true; + } + + /** + * Compare {@link NonNullableStructVector} value equals with given indices and vectors. + * @param left left vector to compare + * @param leftIndex left index + * @param right right vector to compare + * @param rightIndex right index + * @return true if equals, otherwise false + */ + public static boolean compare(NonNullableStructVector left, int leftIndex, NonNullableStructVector right, + int rightIndex) { + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { + return false; + } + + if (!isNull) { + if (!left.getChildFieldNames().equals(right.getChildFieldNames())) { + return false; + } + + for (String name : left.getChildFieldNames()) { + if (!left.getChild(name).equals(leftIndex, right.getChild(name), rightIndex)) { + return false; + } + } + } + return true; + } + + /** + * Check whether indices are valid. + */ + public static boolean checkIndices(ValueVector left, int leftIndex, ValueVector right, int rightIndex) { + if (leftIndex < 0 || rightIndex < 0) { + throw new IllegalStateException(String.format("indices must be non negative, left index: %s, right index: %s", + leftIndex, rightIndex)); + } + + if (leftIndex >= left.getValueCount() || rightIndex >= right.getValueCount()) { + throw new IllegalStateException(String.format("indices mush be less than valueCount, left index: %s, " + + "right index: %s, left valueCount: %s, right valueCount: %s", leftIndex, left.getValueCount(), + rightIndex, right.getValueCount())); + } + + return true; + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index 94640ef3cba..4a976b84e80 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -17,16 +17,13 @@ package org.apache.arrow.vector.compare; -import java.util.ArrayList; import java.util.List; -import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NonNullableStructVector; @@ -77,27 +74,36 @@ public boolean visit(UnionVector left) { } public boolean visit(ZeroVector left) { - return false; + return true; + } + + public boolean visit(ValueVector left) { + throw new UnsupportedOperationException(); + } + + protected boolean compareValueVector(ValueVector left, ValueVector right) { + return left.getField().getType().equals(right.getField().getType()); } protected boolean compareUnionVectors(UnionVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + + if (!compareValueVector(left, right)) { return false; } UnionVector rightVector = (UnionVector) right; - List leftChildrens = left.getChildrenFromFields(); - List rightChildrens = rightVector.getChildrenFromFields(); + List leftChildren = left.getChildrenFromFields(); + List rightChildren = rightVector.getChildrenFromFields(); - if (leftChildrens.size() != rightChildrens.size()) { + if (leftChildren.size() != rightChildren.size()) { return false; } - for (int k = 0; k < leftChildrens.size(); k++) { - RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightChildrens.get(k), + for (int k = 0; k < leftChildren.size(); k++) { + RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightChildren.get(k), leftStart, rightStart, length); - if (!leftChildrens.get(k).accept(visitor)) { + if (!leftChildren.get(k).accept(visitor)) { return false; } } @@ -105,7 +111,7 @@ protected boolean compareUnionVectors(UnionVector left) { } protected boolean compareStructVectors(NonNullableStructVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + if (!compareValueVector(left, right)) { return false; } @@ -115,76 +121,37 @@ protected boolean compareStructVectors(NonNullableStructVector left) { return false; } - List leftChildrens = new ArrayList<>(); - List rightChildrens = new ArrayList<>(); - - for (String child : left.getChildFieldNames()) { - ValueVector v = left.getChild(child); - if (v != null) { - leftChildrens.add(v); - } - } - - for (String child : rightVector.getChildFieldNames()) { - ValueVector v = rightVector.getChild(child); - if (v != null) { - rightChildrens.add(v); - } - } - - if (leftChildrens.size() != rightChildrens.size()) { - return false; - } - - for (int k = 0; k < leftChildrens.size(); k++) { - RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightChildrens.get(k), + for (String name : left.getChildFieldNames()) { + RangeEqualsVisitor visitor = new RangeEqualsVisitor(rightVector.getChild(name), leftStart, rightStart, length); - if (!leftChildrens.get(k).accept(visitor)) { + if (!left.getChild(name).accept(visitor)) { return false; } } + return true; } protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + if (!compareValueVector(left, right)) { return false; } - int typeWidth = left.getTypeWidth(); - for (int i = 0; i < length; i++) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { + if (!CompareUtility.compare(left, leftIndex, (BaseFixedWidthVector) right, rightIndex)) { return false; } - - if (!isNull) { - int start1 = typeWidth * leftIndex; - int end1 = typeWidth * (leftIndex + 1); - - int start2 = typeWidth * rightIndex; - int end2 = typeWidth * (rightIndex + 1); - - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, - right.getDataBuffer(), start2, end2); - - if (ret == 0) { - return false; - } - - } } return true; } protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + if (!compareValueVector(left, right)) { return false; } @@ -192,33 +159,15 @@ protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) int leftIndex = leftStart + i; int rightIndex = rightStart + i; - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { + if (!CompareUtility.compare(left, leftIndex, (BaseVariableWidthVector) right, rightIndex)) { return false; } - - int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; - - if (!isNull) { - final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - - final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, - right.getDataBuffer(), start2, end2); - - if (ret == 0) { - return false; - } - } } return true; } protected boolean compareListVectors(ListVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + if (!compareValueVector(left, right)) { return false; } @@ -226,37 +175,15 @@ protected boolean compareListVectors(ListVector left) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { + if (!CompareUtility.compare(left, leftIndex, (ListVector) right, rightIndex)) { return false; } - - int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; - - if (!isNull) { - final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - - final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - - if ((end1 - start1) != (end2 - start2)) { - return false; - } - - ValueVector dataVector1 = left.getDataVector(); - ValueVector dataVector2 = ((ListVector)right).getDataVector(); - - if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { - return false; - } - } } return true; } protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { - if (!left.getField().getType().equals(right.getField().getType())) { + if (!compareValueVector(left, right)) { return false; } @@ -268,31 +195,9 @@ protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { + if (!CompareUtility.compare(left, leftIndex, (FixedSizeListVector) right, rightIndex)) { return false; } - - int listSize = left.getListSize(); - - if (!isNull) { - final int start1 = leftIndex * listSize; - final int end1 = (leftIndex + 1) * listSize; - - final int start2 = rightIndex * listSize; - final int end2 = (rightIndex + 1) * listSize; - - if ((end1 - start1) != (end2 - start2)) { - return false; - } - - ValueVector dataVector1 = left.getDataVector(); - ValueVector dataVector2 = ((FixedSizeListVector)right).getDataVector(); - - if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { - return false; - } - } } return true; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java index 9b3c240ed78..3d37e543bc7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java @@ -17,13 +17,7 @@ package org.apache.arrow.vector.compare; -import org.apache.arrow.vector.BaseFixedWidthVector; -import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.complex.FixedSizeListVector; -import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.NonNullableStructVector; -import org.apache.arrow.vector.complex.UnionVector; /** * Visitor to compare vectors equal. @@ -35,50 +29,10 @@ public VectorEqualsVisitor(ValueVector right) { } @Override - protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; + protected boolean compareValueVector(ValueVector left, ValueVector right) { + if (!left.getField().equals(right.getField())) { + return false; } - return super.compareBaseFixedWidthVectors(left); - } - - @Override - protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; - } - return super.compareBaseVariableWidthVectors(left); - } - - @Override - protected boolean compareListVectors(ListVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; - } - return super.compareListVectors(left); - } - - @Override - protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; - } - return super.compareFixedSizeListVectors(left); - } - - @Override - protected boolean compareStructVectors(NonNullableStructVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; - } - return super.compareStructVectors(left); - } - - @Override - protected boolean compareUnionVectors(UnionVector left) { - if (left.getValueCount() != right.getValueCount()) { - return false; - } - return super.compareUnionVectors(left); + return left.getValueCount() == right.getValueCount(); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 8d09aa43f56..744a3a92ede 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -39,6 +39,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -525,21 +526,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } - FixedSizeListVector that = (FixedSizeListVector) to; + CompareUtility.checkIndices(this, index, to, toIndex); - boolean isNull = isNull(index); - if (isNull != that.isNull(toIndex)) { - return false; - } + FixedSizeListVector that = (FixedSizeListVector) to; - if (!isNull) { - for (int i = 0; i < listSize; i++) { - if (!vector.equals(index * listSize + i, that, toIndex * listSize + i)) { - return false; - } - } - } - return true; + return CompareUtility.compare(this, index, that, toIndex); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 29c8e43f4fa..8cf50b333fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -34,6 +34,7 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -435,31 +436,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } - ListVector that = (ListVector) to; - - boolean isNull = isNull(index); - if (isNull != that.isNull(toIndex)) { - return false; - } + CompareUtility.checkIndices(this, index, to, toIndex); - if (!isNull) { - final int leftStart = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int leftEnd = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); - - final int rightStart = that.offsetBuffer.getInt(toIndex * OFFSET_WIDTH); - final int rightEnd = that.offsetBuffer.getInt((toIndex + 1) * OFFSET_WIDTH); - - if ((leftEnd - leftStart) != (rightEnd - rightStart)) { - return false; - } + ListVector that = (ListVector) to; - for (int i = 0; i < (leftEnd - leftStart); i++) { - if (!vector.equals(leftStart + i, that.vector, rightStart + i)) { - return false; - } - } - } - return true; + return CompareUtility.compare(this, index, that, toIndex); } private class TransferImpl implements TransferPair { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index 9b457574368..cb6113d5fae 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -29,6 +29,7 @@ import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -312,39 +313,11 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (!this.getField().getType().equals(to.getField().getType())) { return false; } - NonNullableStructVector that = (NonNullableStructVector) to; - - List leftChildrens = new ArrayList<>(); - List rightChildrens = new ArrayList<>(); - - if (!getChildFieldNames().equals(that.getChildFieldNames())) { - return false; - } - - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null) { - leftChildrens.add(v); - } - } + CompareUtility.checkIndices(this, index, to, toIndex); - for (String child : that.getChildFieldNames()) { - ValueVector v = that.getChild(child); - if (v != null) { - rightChildrens.add(v); - } - } - - if (leftChildrens.size() != rightChildrens.size()) { - return false; - } + NonNullableStructVector that = (NonNullableStructVector) to; - for (int i = 0; i < leftChildrens.size(); i++) { - if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) { - return false; - } - } - return true; + return CompareUtility.compare(this, index, that, toIndex); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index a7da368bee4..2ad9fb75091 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -488,57 +488,6 @@ public int hashCode(int index) { } } - @Override - public boolean equals(int index, ValueVector to, int toIndex) { - if (to == null) { - return false; - } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - StructVector that = (StructVector) to; - - boolean isNull = isNull(index); - if (isNull != that.isNull(toIndex)) { - return false; - } - - if (!isNull) { - List leftChildrens = new ArrayList<>(); - List rightChildrens = new ArrayList<>(); - - if (!getChildFieldNames().equals(that.getChildFieldNames())) { - return false; - } - - for (String child : getChildFieldNames()) { - ValueVector v = getChild(child); - if (v != null) { - leftChildrens.add(v); - } - } - - for (String child : that.getChildFieldNames()) { - ValueVector v = that.getChild(child); - if (v != null) { - rightChildrens.add(v); - } - } - - if (leftChildrens.size() != rightChildrens.size()) { - return false; - } - - for (int i = 0; i < leftChildrens.size(); i++) { - if (!leftChildrens.get(i).equals(index, rightChildrens.get(i), toIndex)) { - return false; - } - } - } - - return true; - } - @Override public void get(int index, ComplexHolder holder) { holder.isSet = isSet(index); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 83041cf1d34..ab0efa5adff 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -398,8 +398,8 @@ public void testEncodeUnion() { @Test public void testIntEquals() { //test Int - try (final IntVector vector1 = new IntVector("v1", allocator); - final IntVector vector2 = new IntVector("v2", allocator)) { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -426,8 +426,8 @@ public void testIntEquals() { @Test public void testVarcharEquals() { - try (final VarCharVector vector1 = new VarCharVector("v1", allocator); - final VarCharVector vector2 = new VarCharVector("v2", allocator)) { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -455,8 +455,8 @@ public void testVarcharEquals() { @Test public void testVarBinaryEquals() { - try (final VarBinaryVector vector1 = new VarBinaryVector("v1", allocator); - final VarBinaryVector vector2 = new VarBinaryVector("v2", allocator)) { + try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -484,8 +484,8 @@ public void testVarBinaryEquals() { @Test public void testListEquals() { - try (final ListVector vector1 = ListVector.empty("v1", allocator); - final ListVector vector2 = ListVector.empty("v2", allocator);) { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { Dictionary dict1 = new Dictionary(vector1, new DictionaryEncoding(1L, false, null)); Dictionary dict2 = new Dictionary(vector2, new DictionaryEncoding(1L, false, null)); @@ -514,8 +514,8 @@ public void testListEquals() { @Test public void testStructEquals() { - try (final StructVector vector1 = StructVector.empty("v1", allocator); - final StructVector vector2 = StructVector.empty("v2", allocator);) { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -544,8 +544,8 @@ public void testStructEquals() { @Test public void testUnionEquals() { - try (final UnionVector vector1 = new UnionVector("v1", allocator, null); - final UnionVector vector2 = new UnionVector("v2", allocator, null);) { + try (final UnionVector vector1 = new UnionVector("union", allocator, null); + final UnionVector vector2 = new UnionVector("union", allocator, null);) { final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); uInt4Holder.value = 10; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index fc5c6d42790..cd559a1f7f2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -2255,10 +2255,20 @@ public void testGetNullFromVariableWidthVector() { } } + @Test + public void testZeroVectorEquals() { + try (final ZeroVector vector1 = new ZeroVector(); + final ZeroVector vector2 = new ZeroVector()) { + + VectorEqualsVisitor visitor = new VectorEqualsVisitor(vector2); + assertTrue(vector1.accept(visitor)); + } + } + @Test public void testIntVectorEqualsWithNull() { - try (final IntVector vector1 = new IntVector("v1", allocator); - final IntVector vector2 = new IntVector("v2", allocator)) { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { vector1.allocateNew(2); vector1.setValueCount(2); @@ -2277,8 +2287,8 @@ public void testIntVectorEqualsWithNull() { @Test public void testIntVectorEquals() { - try (final IntVector vector1 = new IntVector("v1", allocator); - final IntVector vector2 = new IntVector("v2", allocator)) { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { vector1.allocateNew(3); vector1.setValueCount(3); @@ -2307,9 +2317,9 @@ public void testIntVectorEquals() { @Test public void testDecimalVectorEquals() { - try (final DecimalVector vector1 = new DecimalVector("v1", allocator, 3, 3); - final DecimalVector vector2 = new DecimalVector("v2", allocator, 3, 3); - final DecimalVector vector3 = new DecimalVector("v3", allocator, 3, 2)) { + try (final DecimalVector vector1 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector2 = new DecimalVector("decimal", allocator, 3, 3); + final DecimalVector vector3 = new DecimalVector("decimal", allocator, 3, 2)) { vector1.allocateNew(2); vector1.setValueCount(2); @@ -2337,8 +2347,8 @@ public void testDecimalVectorEquals() { @Test public void testVarcharVectorEuqalsWithNull() { - try (final VarCharVector vector1 = new VarCharVector("v1", allocator); - final VarCharVector vector2 = new VarCharVector("v2", allocator)) { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { vector1.allocateNew(); vector2.allocateNew(); @@ -2358,8 +2368,8 @@ public void testVarcharVectorEuqalsWithNull() { @Test public void testVarcharVectorEquals() { - try (final VarCharVector vector1 = new VarCharVector("v1", allocator); - final VarCharVector vector2 = new VarCharVector("v2", allocator)) { + try (final VarCharVector vector1 = new VarCharVector("varchar", allocator); + final VarCharVector vector2 = new VarCharVector("varchar", allocator)) { vector1.allocateNew(); vector2.allocateNew(); @@ -2385,8 +2395,8 @@ public void testVarcharVectorEquals() { @Test public void testVarBinaryVectorEquals() { - try (final VarBinaryVector vector1 = new VarBinaryVector("v1", allocator); - final VarBinaryVector vector2 = new VarBinaryVector("v2", allocator)) { + try (final VarBinaryVector vector1 = new VarBinaryVector("binary", allocator); + final VarBinaryVector vector2 = new VarBinaryVector("binary", allocator)) { vector1.allocateNew(); vector2.allocateNew(); @@ -2412,8 +2422,8 @@ public void testVarBinaryVectorEquals() { @Test public void testListVectorEqualsWithNull() { - try (final ListVector vector1 = ListVector.empty("v1", allocator); - final ListVector vector2 = ListVector.empty("v2", allocator);) { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { UnionListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -2440,8 +2450,8 @@ public void testListVectorEqualsWithNull() { @Test public void testListVectorEquals() { - try (final ListVector vector1 = ListVector.empty("v1", allocator); - final ListVector vector2 = ListVector.empty("v2", allocator);) { + try (final ListVector vector1 = ListVector.empty("list", allocator); + final ListVector vector2 = ListVector.empty("list", allocator);) { UnionListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -2473,8 +2483,8 @@ public void testListVectorEquals() { @Test public void testStructVectorEqualsWithNull() { - try (final StructVector vector1 = StructVector.empty("v1", allocator); - final StructVector vector2 = StructVector.empty("v2", allocator);) { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -2502,8 +2512,8 @@ public void testStructVectorEqualsWithNull() { @Test public void testStructVectorEquals() { - try (final StructVector vector1 = StructVector.empty("v1", allocator); - final StructVector vector2 = StructVector.empty("v2", allocator);) { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -2536,8 +2546,8 @@ public void testStructVectorEquals() { @Test public void testStructVectorEqualsWithDiffChild() { - try (final StructVector vector1 = StructVector.empty("v1", allocator); - final StructVector vector2 = StructVector.empty("v2", allocator);) { + try (final StructVector vector1 = StructVector.empty("struct", allocator); + final StructVector vector2 = StructVector.empty("struct", allocator);) { vector1.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); vector1.addOrGet("f1", FieldType.nullable(new ArrowType.Int(64, true)), BigIntVector.class); vector2.addOrGet("f0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); @@ -2564,8 +2574,8 @@ public void testStructVectorEqualsWithDiffChild() { @Test public void testUnionVectorEquals() { - try (final UnionVector vector1 = new UnionVector("v1", allocator, null); - final UnionVector vector2 = new UnionVector("v2", allocator, null);) { + try (final UnionVector vector1 = new UnionVector("union", allocator, null); + final UnionVector vector2 = new UnionVector("union", allocator, null);) { final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder(); uInt4Holder.value = 10; @@ -2594,6 +2604,26 @@ public void testUnionVectorEquals() { } } + @Test(expected = IllegalStateException.class) + public void testEqualsWithIndexOutOfRange() { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { + + vector1.allocateNew(2); + vector1.setValueCount(2); + vector2.allocateNew(2); + vector2.setValueCount(2); + + vector1.setSafe(0, 1); + vector1.setSafe(1, 2); + + vector2.setSafe(0, 1); + vector2.setSafe(1, 2); + + assertTrue(vector1.equals(3, vector2, 2)); + } + } + private void writeStructVector(NullableStructWriter writer, int value1, long value2) { writer.start(); writer.integer("f0").writeInt(value1); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 21530985650..792bd29903b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -36,7 +36,6 @@ import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; @@ -216,11 +215,6 @@ public boolean equals(int index, ValueVector to, int toIndex) { return getUnderlyingVector().equals(index, to, toIndex); } - @Override - public boolean accept(RangeEqualsVisitor visitor) { - return visitor.visit(getUnderlyingVector()); - } - public void set(int index, UUID uuid) { ByteBuffer bb = ByteBuffer.allocate(16); bb.putLong(uuid.getMostSignificantBits()); From a5d22fd4f6b8e2402ab9a6408772076a324a7b51 Mon Sep 17 00:00:00 2001 From: tianchen Date: Fri, 9 Aug 2019 17:19:40 +0800 Subject: [PATCH 12/13] fix variable names --- .../arrow/vector/compare/CompareUtility.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java index 8bab136c8f1..4fd4fdcc4c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java @@ -85,14 +85,14 @@ public static boolean compare(BaseVariableWidthVector left, int leftIndex, int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; if (!isNull) { - final int start1 = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int end1 = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - final int start2 = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int end2 = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), start1, end1, - right.getDataBuffer(), start2, end2); + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, + right.getDataBuffer(), startByteRight, endByteRight); if (ret == 0) { return false; @@ -129,10 +129,10 @@ public static boolean compare(ListVector left, int leftIndex, ListVector right, return false; } - ValueVector dataVector1 = left.getDataVector(); - ValueVector dataVector2 = right.getDataVector(); + ValueVector leftDataVector = left.getDataVector(); + ValueVector rightDataVector = right.getDataVector(); - if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, startByteLeft, + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, startByteRight, (endByteLeft - startByteLeft)))) { return false; } @@ -157,20 +157,21 @@ public static boolean compare(FixedSizeListVector left, int leftIndex, FixedSize int listSize = left.getListSize(); if (!isNull) { - final int start1 = leftIndex * listSize; - final int end1 = (leftIndex + 1) * listSize; + final int startByteLeft = leftIndex * listSize; + final int endByteLeft = (leftIndex + 1) * listSize; - final int start2 = rightIndex * listSize; - final int end2 = (rightIndex + 1) * listSize; + final int startByteRight = rightIndex * listSize; + final int endByteRight = (rightIndex + 1) * listSize; - if ((end1 - start1) != (end2 - start2)) { + if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { return false; } - ValueVector dataVector1 = left.getDataVector(); - ValueVector dataVector2 = right.getDataVector(); + ValueVector leftDataVector = left.getDataVector(); + ValueVector rightDataVector = right.getDataVector(); - if (!dataVector1.accept(new RangeEqualsVisitor(dataVector2, start1, start2, (end1 - start1)))) { + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, startByteRight, + (endByteLeft - startByteLeft)))) { return false; } } From 7e20f79d5a74f0d2026f186db431083762519eec Mon Sep 17 00:00:00 2001 From: tianchen Date: Sat, 10 Aug 2019 09:48:26 +0800 Subject: [PATCH 13/13] remove CompareUtility --- .../main/codegen/templates/UnionVector.java | 21 +- .../arrow/vector/BaseFixedWidthVector.java | 15 +- .../arrow/vector/BaseVariableWidthVector.java | 15 +- .../arrow/vector/compare/CompareUtility.java | 227 ------------------ .../vector/compare/RangeEqualsVisitor.java | 93 ++++++- .../vector/compare/VectorEqualsVisitor.java | 5 +- .../vector/complex/FixedSizeListVector.java | 15 +- .../arrow/vector/complex/ListVector.java | 16 +- .../complex/NonNullableStructVector.java | 15 +- .../apache/arrow/vector/TestValueVector.java | 2 +- 10 files changed, 131 insertions(+), 293 deletions(-) delete mode 100644 java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 2880d9eb32f..2c762dfeca8 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -18,6 +18,7 @@ import io.netty.buffer.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.types.UnionMode; import org.apache.arrow.vector.compare.RangeEqualsVisitor; @@ -37,7 +38,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.util.CallBack; @@ -668,20 +668,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - CompareUtility.checkIndices(this, index, to, toIndex); + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - UnionVector that = (UnionVector) to; - ValueVector leftVector = getVector(index); - ValueVector rightVector = that.getVector(toIndex); - - if (leftVector == null) { - return rightVector == null; - } else { - return leftVector.equals(index, rightVector, toIndex); - } + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 8ba32b848cc..4e953a345cd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -26,7 +26,6 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; @@ -877,15 +876,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - - CompareUtility.checkIndices(this, index, to, toIndex); - - BaseFixedWidthVector that = (BaseFixedWidthVector) to; + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - return CompareUtility.compare(this, index, that, toIndex); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 67edde4e68c..b7aa8161bd0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -28,7 +28,6 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; @@ -1371,15 +1370,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - - CompareUtility.checkIndices(this, index, to, toIndex); - - BaseVariableWidthVector that = (BaseVariableWidthVector) to; + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - return CompareUtility.compare(this, index, that, toIndex); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java deleted file mode 100644 index 4fd4fdcc4c0..00000000000 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/CompareUtility.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.vector.compare; - -import org.apache.arrow.memory.util.ByteFunctionHelpers; -import org.apache.arrow.vector.BaseFixedWidthVector; -import org.apache.arrow.vector.BaseVariableWidthVector; -import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.complex.BaseRepeatedValueVector; -import org.apache.arrow.vector.complex.FixedSizeListVector; -import org.apache.arrow.vector.complex.ListVector; -import org.apache.arrow.vector.complex.NonNullableStructVector; - -/** - * Utility to compare two single values in two vectors with same type. - */ -public class CompareUtility { - - /** - * Compare {@link BaseFixedWidthVector} value equals with given indices and vectors. - * @param left left vector to compare - * @param leftIndex left index - * @param right right vector to compare - * @param rightIndex right index - * @return true if equals, otherwise false - */ - public static boolean compare(BaseFixedWidthVector left, int leftIndex, BaseFixedWidthVector right, int rightIndex) { - - boolean isNull = left.isNull(leftIndex); - - if (isNull != right.isNull(rightIndex)) { - return false; - } - - int typeWidth = left.getTypeWidth(); - if (!isNull) { - int startByteLeft = typeWidth * leftIndex; - int endByteLeft = typeWidth * (leftIndex + 1); - - int startByteRight = typeWidth * rightIndex; - int endByteRight = typeWidth * (rightIndex + 1); - - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, - right.getDataBuffer(), startByteRight, endByteRight); - - if (ret == 0) { - return false; - } - - } - return true; - } - - /** - * Compare {@link BaseVariableWidthVector} value equals with given indices and vectors. - * @param left left vector to compare - * @param leftIndex left index - * @param right right vector to compare - * @param rightIndex right index - * @return true if equals, otherwise false - */ - public static boolean compare(BaseVariableWidthVector left, int leftIndex, - BaseVariableWidthVector right, int rightIndex) { - - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { - return false; - } - - int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; - - if (!isNull) { - final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - - final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - - int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, - right.getDataBuffer(), startByteRight, endByteRight); - - if (ret == 0) { - return false; - } - } - return true; - } - - /** - * Compare {@link ListVector} value equals with given indices and vectors. - * @param left left vector to compare - * @param leftIndex left index - * @param right right vector to compare - * @param rightIndex right index - * @return true if equals, otherwise false - */ - public static boolean compare(ListVector left, int leftIndex, ListVector right, int rightIndex) { - - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { - return false; - } - - int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; - - if (!isNull) { - final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); - - final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); - final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); - - if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { - return false; - } - - ValueVector leftDataVector = left.getDataVector(); - ValueVector rightDataVector = right.getDataVector(); - - if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, - startByteRight, (endByteLeft - startByteLeft)))) { - return false; - } - } - return true; - } - - /** - * Compare {@link FixedSizeListVector} value equals with given indices and vectors. - * @param left left vector to compare - * @param leftIndex left index - * @param right right vector to compare - * @param rightIndex right index - * @return true if equals, otherwise false - */ - public static boolean compare(FixedSizeListVector left, int leftIndex, FixedSizeListVector right, int rightIndex) { - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { - return false; - } - - int listSize = left.getListSize(); - - if (!isNull) { - final int startByteLeft = leftIndex * listSize; - final int endByteLeft = (leftIndex + 1) * listSize; - - final int startByteRight = rightIndex * listSize; - final int endByteRight = (rightIndex + 1) * listSize; - - if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { - return false; - } - - ValueVector leftDataVector = left.getDataVector(); - ValueVector rightDataVector = right.getDataVector(); - - if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, startByteRight, - (endByteLeft - startByteLeft)))) { - return false; - } - } - return true; - } - - /** - * Compare {@link NonNullableStructVector} value equals with given indices and vectors. - * @param left left vector to compare - * @param leftIndex left index - * @param right right vector to compare - * @param rightIndex right index - * @return true if equals, otherwise false - */ - public static boolean compare(NonNullableStructVector left, int leftIndex, NonNullableStructVector right, - int rightIndex) { - boolean isNull = left.isNull(leftIndex); - if (isNull != right.isNull(rightIndex)) { - return false; - } - - if (!isNull) { - if (!left.getChildFieldNames().equals(right.getChildFieldNames())) { - return false; - } - - for (String name : left.getChildFieldNames()) { - if (!left.getChild(name).equals(leftIndex, right.getChild(name), rightIndex)) { - return false; - } - } - } - return true; - } - - /** - * Check whether indices are valid. - */ - public static boolean checkIndices(ValueVector left, int leftIndex, ValueVector right, int rightIndex) { - if (leftIndex < 0 || rightIndex < 0) { - throw new IllegalStateException(String.format("indices must be non negative, left index: %s, right index: %s", - leftIndex, rightIndex)); - } - - if (leftIndex >= left.getValueCount() || rightIndex >= right.getValueCount()) { - throw new IllegalStateException(String.format("indices mush be less than valueCount, left index: %s, " + - "right index: %s, left valueCount: %s, right valueCount: %s", leftIndex, left.getValueCount(), - rightIndex, right.getValueCount())); - } - - return true; - } -} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index 4a976b84e80..19cf79ed658 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -19,11 +19,13 @@ import java.util.List; +import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.NonNullableStructVector; @@ -142,11 +144,28 @@ protected boolean compareBaseFixedWidthVectors(BaseFixedWidthVector left) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - if (!CompareUtility.compare(left, leftIndex, (BaseFixedWidthVector) right, rightIndex)) { + boolean isNull = left.isNull(leftIndex); + + if (isNull != right.isNull(rightIndex)) { return false; } - } + int typeWidth = left.getTypeWidth(); + if (!isNull) { + int startByteLeft = typeWidth * leftIndex; + int endByteLeft = typeWidth * (leftIndex + 1); + + int startByteRight = typeWidth * rightIndex; + int endByteRight = typeWidth * (rightIndex + 1); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, + right.getDataBuffer(), startByteRight, endByteRight); + + if (ret == 0) { + return false; + } + } + } return true; } @@ -159,9 +178,27 @@ protected boolean compareBaseVariableWidthVectors(BaseVariableWidthVector left) int leftIndex = leftStart + i; int rightIndex = rightStart + i; - if (!CompareUtility.compare(left, leftIndex, (BaseVariableWidthVector) right, rightIndex)) { + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { return false; } + + int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; + + if (!isNull) { + final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + int ret = ByteFunctionHelpers.equal(left.getDataBuffer(), startByteLeft, endByteLeft, + right.getDataBuffer(), startByteRight, endByteRight); + + if (ret == 0) { + return false; + } + } } return true; } @@ -175,9 +212,32 @@ protected boolean compareListVectors(ListVector left) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - if (!CompareUtility.compare(left, leftIndex, (ListVector) right, rightIndex)) { + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { return false; } + + int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; + + if (!isNull) { + final int startByteLeft = left.getOffsetBuffer().getInt(leftIndex * offsetWidth); + final int endByteLeft = left.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + + final int startByteRight = right.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int endByteRight = right.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + + if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { + return false; + } + + ValueVector leftDataVector = left.getDataVector(); + ValueVector rightDataVector = ((ListVector)right).getDataVector(); + + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, + startByteRight, (endByteLeft - startByteLeft)))) { + return false; + } + } } return true; } @@ -195,9 +255,32 @@ protected boolean compareFixedSizeListVectors(FixedSizeListVector left) { int leftIndex = leftStart + i; int rightIndex = rightStart + i; - if (!CompareUtility.compare(left, leftIndex, (FixedSizeListVector) right, rightIndex)) { + boolean isNull = left.isNull(leftIndex); + if (isNull != right.isNull(rightIndex)) { return false; } + + int listSize = left.getListSize(); + + if (!isNull) { + final int startByteLeft = leftIndex * listSize; + final int endByteLeft = (leftIndex + 1) * listSize; + + final int startByteRight = rightIndex * listSize; + final int endByteRight = (rightIndex + 1) * listSize; + + if ((endByteLeft - startByteLeft) != (endByteRight - startByteRight)) { + return false; + } + + ValueVector leftDataVector = left.getDataVector(); + ValueVector rightDataVector = ((FixedSizeListVector)right).getDataVector(); + + if (!leftDataVector.accept(new RangeEqualsVisitor(rightDataVector, startByteLeft, startByteRight, + (endByteLeft - startByteLeft)))) { + return false; + } + } } return true; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java index 3d37e543bc7..47071dd1958 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorEqualsVisitor.java @@ -17,6 +17,7 @@ package org.apache.arrow.vector.compare; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.ValueVector; /** @@ -25,12 +26,12 @@ public class VectorEqualsVisitor extends RangeEqualsVisitor { public VectorEqualsVisitor(ValueVector right) { - super(right, 0, 0, right.getValueCount()); + super(Preconditions.checkNotNull(right), 0, 0, right.getValueCount()); } @Override protected boolean compareValueVector(ValueVector left, ValueVector right) { - if (!left.getField().equals(right.getField())) { + if (!super.compareValueVector(left, right)) { return false; } return left.getValueCount() == right.getValueCount(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 744a3a92ede..f343fd594f6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -39,7 +39,6 @@ import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; @@ -522,15 +521,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - - CompareUtility.checkIndices(this, index, to, toIndex); - - FixedSizeListVector that = (FixedSizeListVector) to; + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - return CompareUtility.compare(this, index, that, toIndex); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 8cf50b333fd..f07fbdb3e7b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -28,13 +28,13 @@ import org.apache.arrow.memory.BaseAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.AddOrGetResult; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.BufferBacked; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.ComplexCopier; import org.apache.arrow.vector.complex.impl.UnionListReader; @@ -432,15 +432,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - - CompareUtility.checkIndices(this, index, to, toIndex); - - ListVector that = (ListVector) to; + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - return CompareUtility.compare(this, index, that, toIndex); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } private class TransferImpl implements TransferPair { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java index cb6113d5fae..995751ed0b3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java @@ -26,10 +26,10 @@ import java.util.Map; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.DensityAwareVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.compare.CompareUtility; import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; @@ -310,14 +310,13 @@ public boolean equals(int index, ValueVector to, int toIndex) { if (to == null) { return false; } - if (!this.getField().getType().equals(to.getField().getType())) { - return false; - } - CompareUtility.checkIndices(this, index, to, toIndex); - - NonNullableStructVector that = (NonNullableStructVector) to; + Preconditions.checkArgument(index >= 0 && index < valueCount, + "index %s out of range[0, %s]:", index, valueCount - 1); + Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(), + "index %s out of range[0, %s]:", index, to.getValueCount() - 1); - return CompareUtility.compare(this, index, that, toIndex); + RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1); + return this.accept(visitor); } @Override diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index cd559a1f7f2..b553f316db3 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -2604,7 +2604,7 @@ public void testUnionVectorEquals() { } } - @Test(expected = IllegalStateException.class) + @Test(expected = IllegalArgumentException.class) public void testEqualsWithIndexOutOfRange() { try (final IntVector vector1 = new IntVector("int", allocator); final IntVector vector2 = new IntVector("int", allocator)) {