From fbc46a6f5ec2a4aaf3c6b4d5d776ccc7b114842f Mon Sep 17 00:00:00 2001 From: Pete Robbins Date: Wed, 21 Dec 2016 10:06:41 +0000 Subject: [PATCH 1/5] o.a.s.unsafe.types.UTF8StringSuite.writeToOutputStreamIntArray test fails on big endian. Only change byte order on little endian --- .../java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 04f684577dfd8..9647ba3f2bc46 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -591,7 +591,11 @@ public void writeToOutputStreamIntArray() throws IOException { // verify that writes work on objects that are not byte arrays final ByteBuffer buffer = StandardCharsets.UTF_8.encode("大千世界"); buffer.position(0); - buffer.order(ByteOrder.LITTLE_ENDIAN); + + // ByteBuffer is BIG_ENDIAN by default + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + } final int length = buffer.limit(); assertEquals(12, length); From 30e20be2c199cc57a6a85547770dfa6fc3d32752 Mon Sep 17 00:00:00 2001 From: Pete Robbins Date: Thu, 22 Dec 2016 09:14:14 +0000 Subject: [PATCH 2/5] Simplify setting of byte order --- .../java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 9647ba3f2bc46..6743cd8c919ac 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -592,10 +592,8 @@ public void writeToOutputStreamIntArray() throws IOException { final ByteBuffer buffer = StandardCharsets.UTF_8.encode("大千世界"); buffer.position(0); - // ByteBuffer is BIG_ENDIAN by default - if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { - buffer.order(ByteOrder.LITTLE_ENDIAN); - } + // Set buffer to native byte order + buffer.order(ByteOrder.nativeOrder()); final int length = buffer.limit(); assertEquals(12, length); From 1bc1adf48dae6b1047ff4d4e3d467ffec88abe12 Mon Sep 17 00:00:00 2001 From: Pete Robbins Date: Fri, 23 Dec 2016 08:56:42 +0000 Subject: [PATCH 3/5] remove redundant comment --- .../java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 6743cd8c919ac..6f6e0ef0e4855 100644 --- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -591,8 +591,6 @@ public void writeToOutputStreamIntArray() throws IOException { // verify that writes work on objects that are not byte arrays final ByteBuffer buffer = StandardCharsets.UTF_8.encode("大千世界"); buffer.position(0); - - // Set buffer to native byte order buffer.order(ByteOrder.nativeOrder()); final int length = buffer.limit(); From 950415f98c4574532da9dd089e5a9b027d3683d8 Mon Sep 17 00:00:00 2001 From: Pete Robbins Date: Thu, 23 Feb 2017 11:33:53 +0000 Subject: [PATCH 4/5] Update tests to produce reliably ordered results --- .../inputs/subquery/in-subquery/in-joins.sql | 2 +- .../inputs/subquery/in-subquery/in-set-operations.sql | 6 +++--- .../inputs/subquery/in-subquery/not-in-joins.sql | 5 +++-- .../results/subquery/in-subquery/in-joins.sql.out | 2 +- .../subquery/in-subquery/in-set-operations.sql.out | 10 +++++----- .../results/subquery/in-subquery/not-in-joins.sql.out | 11 ++++++----- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index b10c41929cdaf..880175fd7add0 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -79,7 +79,7 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC; +ORDER BY t1a DESC, t3b DESC; -- TC 01.03 SELECT Count(DISTINCT(t1a)) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-set-operations.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-set-operations.sql index 6b9e8bf2f362d..5c371d2305ac8 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-set-operations.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-set-operations.sql @@ -287,7 +287,7 @@ WHERE t1a IN (SELECT t3a WHERE t1b > 6) AS t5) GROUP BY t1a, t1b, t1c, t1d HAVING t1c IS NOT NULL AND t1b IS NOT NULL -ORDER BY t1c DESC; +ORDER BY t1c DESC, t1a DESC; -- TC 01.08 SELECT t1a, @@ -351,7 +351,7 @@ WHERE t1b IN FROM t1 WHERE t1b > 6) AS t4 WHERE t2b = t1b) -ORDER BY t1c DESC NULLS last; +ORDER BY t1c DESC NULLS last, t1a DESC; -- TC 01.11 SELECT * @@ -468,5 +468,5 @@ HAVING t1b NOT IN EXCEPT SELECT t3b FROM t3) -ORDER BY t1c DESC NULLS LAST; +ORDER BY t1c DESC NULLS LAST, t1i; diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index 505366b7acd43..55e9862d42600 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -85,7 +85,7 @@ AND t1b != t3b AND t1d = t2d GROUP BY t1a, t1b, t1c, t3a, t3b, t3c HAVING count(distinct(t3a)) >= 1 -ORDER BY t1a; +ORDER BY t1a, t3b; -- TC 01.03 SELECT t1a, @@ -106,7 +106,8 @@ AND t1d NOT IN SELECT t2d FROM t2 RIGHT JOIN t1 on t2e = t1e - WHERE t1a = t2a); + WHERE t1a = t2a) +ORDER BY t1a, t1h; -- TC 01.04 SELECT Count(DISTINCT( t1a )), diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out index 7258bcfc6ab72..ab6a11a2b7efa 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-joins.sql.out @@ -102,7 +102,7 @@ GROUP BY t1a, t3a, t3b, t3c -ORDER BY t1a DESC +ORDER BY t1a DESC, t3b DESC -- !query 4 schema struct -- !query 4 output diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-set-operations.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-set-operations.sql.out index 878bc755ef5fc..e06f9206d3401 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-set-operations.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-set-operations.sql.out @@ -353,7 +353,7 @@ WHERE t1a IN (SELECT t3a WHERE t1b > 6) AS t5) GROUP BY t1a, t1b, t1c, t1d HAVING t1c IS NOT NULL AND t1b IS NOT NULL -ORDER BY t1c DESC +ORDER BY t1c DESC, t1a DESC -- !query 9 schema struct -- !query 9 output @@ -445,7 +445,7 @@ WHERE t1b IN FROM t1 WHERE t1b > 6) AS t4 WHERE t2b = t1b) -ORDER BY t1c DESC NULLS last +ORDER BY t1c DESC NULLS last, t1a DESC -- !query 12 schema struct -- !query 12 output @@ -580,16 +580,16 @@ HAVING t1b NOT IN EXCEPT SELECT t3b FROM t3) -ORDER BY t1c DESC NULLS LAST +ORDER BY t1c DESC NULLS LAST, t1i -- !query 15 schema struct -- !query 15 output -1 8 16 2014-05-05 1 8 16 2014-05-04 +1 8 16 2014-05-05 1 16 12 2014-06-04 1 16 12 2014-07-04 1 6 8 2014-04-04 +1 10 NULL 2014-05-04 1 10 NULL 2014-08-04 1 10 NULL 2014-09-04 1 10 NULL 2015-05-04 -1 10 NULL 2014-05-04 diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out index db01fa455735c..547696f76a607 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out @@ -112,12 +112,12 @@ AND t1b != t3b AND t1d = t2d GROUP BY t1a, t1b, t1c, t3a, t3b, t3c HAVING count(distinct(t3a)) >= 1 -ORDER BY t1a +ORDER BY t1a, t3b -- !query 4 schema struct -- !query 4 output -val1c 8 16 1 10 12 val1c 8 16 1 6 12 +val1c 8 16 1 10 12 val1c 8 16 1 17 16 @@ -141,15 +141,16 @@ AND t1d NOT IN FROM t2 RIGHT JOIN t1 on t2e = t1e WHERE t1a = t2a) +ORDER BY t1a, t1h -- !query 5 schema struct -- !query 5 output -val1a 16 12 10 2014-07-04 01:01:00 -val1a 16 12 21 2014-06-04 01:02:00.001 val1a 6 8 10 2014-04-04 01:00:00 val1a 6 8 10 2014-04-04 01:02:00.001 -val1d 10 NULL 12 2015-05-04 01:01:00 +val1a 16 12 21 2014-06-04 01:02:00.001 +val1a 16 12 10 2014-07-04 01:01:00 val1d NULL 16 22 2014-06-04 01:01:00 +val1d 10 NULL 12 2015-05-04 01:01:00 val1e 10 NULL 25 2014-08-04 01:01:00 From 4a4d7ad4b349e49dc4cb81235f796e360dd183f8 Mon Sep 17 00:00:00 2001 From: Pete Robbins Date: Mon, 27 Feb 2017 10:06:50 +0000 Subject: [PATCH 5/5] remove unnecessary ordering --- .../sql-tests/inputs/subquery/in-subquery/not-in-joins.sql | 3 +-- .../results/subquery/in-subquery/not-in-joins.sql.out | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index 55e9862d42600..e09b91f18de0a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -106,8 +106,7 @@ AND t1d NOT IN SELECT t2d FROM t2 RIGHT JOIN t1 on t2e = t1e - WHERE t1a = t2a) -ORDER BY t1a, t1h; + WHERE t1a = t2a); -- TC 01.04 SELECT Count(DISTINCT( t1a )), diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out index 547696f76a607..bae5d00cc8632 100644 --- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/not-in-joins.sql.out @@ -141,16 +141,15 @@ AND t1d NOT IN FROM t2 RIGHT JOIN t1 on t2e = t1e WHERE t1a = t2a) -ORDER BY t1a, t1h -- !query 5 schema struct -- !query 5 output +val1a 16 12 10 2014-07-04 01:01:00 +val1a 16 12 21 2014-06-04 01:02:00.001 val1a 6 8 10 2014-04-04 01:00:00 val1a 6 8 10 2014-04-04 01:02:00.001 -val1a 16 12 21 2014-06-04 01:02:00.001 -val1a 16 12 10 2014-07-04 01:01:00 -val1d NULL 16 22 2014-06-04 01:01:00 val1d 10 NULL 12 2015-05-04 01:01:00 +val1d NULL 16 22 2014-06-04 01:01:00 val1e 10 NULL 25 2014-08-04 01:01:00