From f5ecab6431951fa6314a36f6f73bd50562bb31bc Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Sun, 17 Jan 2016 18:08:27 +0100 Subject: [PATCH 1/9] Change parsed decimal literal datatype from Double to Decimal --- .../spark/sql/catalyst/CatalystQl.scala | 4 ++ .../catalyst/analysis/HiveTypeCoercion.scala | 3 +- .../spark/sql/MathExpressionsSuite.scala | 12 ++++- .../org/apache/spark/sql/SQLQuerySuite.scala | 50 +++++++++---------- .../datasources/json/JsonSuite.scala | 4 +- .../execution/HiveCompatibilitySuite.scala | 13 +++-- 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala index c1591ecfe2b4d..48560e4e4fcb1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala @@ -623,6 +623,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C val CASE = "(?i)CASE".r val INTEGRAL = "[+-]?\\d+".r + val DECIMAL = "[+-]?((\\d+(\\.\\d*)?)|(\\.\\d+))".r protected def nodeToExpr(node: ASTNode): Expression = node match { /* Attribute References */ @@ -799,7 +800,10 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C Literal(v.longValue()) case v => Literal(v.underlying()) } + case DECIMAL(_*) => + Literal(BigDecimal(text).underlying()) case _ => + // Convert a scientifically notated decimal into a double. Literal(text.toDouble) } case ast if ast.tokenType == SparkSqlParser.StringLiteral => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 2737fe32cd086..c712501ac28ad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -651,12 +651,11 @@ object HiveTypeCoercion { case e if !e.childrenResolved => e // Find tightest common type for If, if the true value and false value have different types. case i @ If(pred, left, right) if left.dataType != right.dataType => - findTightestCommonTypeToString(left.dataType, right.dataType).map { widestType => + findWiderTypeForTwo(left.dataType, right.dataType).map { widestType => val newLeft = if (left.dataType == widestType) left else Cast(left, widestType) val newRight = if (right.dataType == widestType) right else Cast(right, widestType) If(pred, newLeft, newRight) }.getOrElse(i) // If there is no applicable conversion, leave expression unchanged. - // Convert If(null literal, _, _) into boolean type. // In the optimizer, we should short-circuit this directly into false value. case If(pred, left, right) if pred.dataType == NullType => diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala index aec450e0a6084..013a90875e2b2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala @@ -212,7 +212,7 @@ class MathExpressionsSuite extends QueryTest with SharedSQLContext { Seq(Row(5, 10, 0), Row(55, 60, 100), Row(555, 560, 600)) ) - val pi = "3.1415BD" + val pi = "3.1415" checkAnswer( sql(s"SELECT round($pi, -3), round($pi, -2), round($pi, -1), " + s"round($pi, 0), round($pi, 1), round($pi, 2), round($pi, 3)"), @@ -367,6 +367,16 @@ class MathExpressionsSuite extends QueryTest with SharedSQLContext { checkAnswer( input.toDF("key", "value").selectExpr("abs(key) a").sort("a"), input.map(pair => Row(pair._2))) + + checkAnswer( + sql("select abs(0), abs(-1), abs(123), abs(-9223372036854775807), abs(9223372036854775807)"), + Row(0, 1, 123, 9223372036854775807L, 9223372036854775807L) + ) + + checkAnswer( + sql("select abs(0.0), abs(-3.14159265), abs(3.14159265)"), + Row(BigDecimal("0.0"), BigDecimal("3.14159265"), BigDecimal("3.14159265")) + ) } test("log2") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index bdb9421cc19d7..5483a7e147c47 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -1193,19 +1193,19 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { test("Floating point number format") { checkAnswer( - sql("SELECT 0.3"), Row(0.3) + sql("SELECT 0.3"), Row(BigDecimal(0.3)) ) checkAnswer( - sql("SELECT -0.8"), Row(-0.8) + sql("SELECT -0.8"), Row(BigDecimal(-0.8)) ) checkAnswer( - sql("SELECT .5"), Row(0.5) + sql("SELECT .5"), Row(BigDecimal(0.5)) ) checkAnswer( - sql("SELECT -.18"), Row(-0.18) + sql("SELECT -.18"), Row(BigDecimal(-0.18)) ) } @@ -1219,11 +1219,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { ) checkAnswer( - sql("SELECT 9223372036854775808BD"), Row(new java.math.BigDecimal("9223372036854775808")) + sql("SELECT 9223372036854775808"), Row(new java.math.BigDecimal("9223372036854775808")) ) checkAnswer( - sql("SELECT -9223372036854775809BD"), Row(new java.math.BigDecimal("-9223372036854775809")) + sql("SELECT -9223372036854775809"), Row(new java.math.BigDecimal("-9223372036854775809")) ) } @@ -1238,11 +1238,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { ) checkAnswer( - sql("SELECT -5.2BD"), Row(BigDecimal(-5.2)) + sql("SELECT -5.2"), Row(BigDecimal(-5.2)) ) checkAnswer( - sql("SELECT +6.8"), Row(6.8d) + sql("SELECT +6.8e0"), Row(6.8d) ) checkAnswer( @@ -1617,20 +1617,20 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("decimal precision with multiply/division") { - checkAnswer(sql("select 10.3BD * 3.0BD"), Row(BigDecimal("30.90"))) - checkAnswer(sql("select 10.3000BD * 3.0BD"), Row(BigDecimal("30.90000"))) - checkAnswer(sql("select 10.30000BD * 30.0BD"), Row(BigDecimal("309.000000"))) - checkAnswer(sql("select 10.300000000000000000BD * 3.000000000000000000BD"), + checkAnswer(sql("select 10.3 * 3.0"), Row(BigDecimal("30.90"))) + checkAnswer(sql("select 10.3000 * 3.0"), Row(BigDecimal("30.90000"))) + checkAnswer(sql("select 10.30000 * 30.0"), Row(BigDecimal("309.000000"))) + checkAnswer(sql("select 10.300000000000000000 * 3.000000000000000000"), Row(BigDecimal("30.900000000000000000000000000000000000", new MathContext(38)))) - checkAnswer(sql("select 10.300000000000000000BD * 3.0000000000000000000BD"), + checkAnswer(sql("select 10.300000000000000000 * 3.0000000000000000000"), Row(null)) - checkAnswer(sql("select 10.3BD / 3.0BD"), Row(BigDecimal("3.433333"))) - checkAnswer(sql("select 10.3000BD / 3.0BD"), Row(BigDecimal("3.4333333"))) - checkAnswer(sql("select 10.30000BD / 30.0BD"), Row(BigDecimal("0.343333333"))) - checkAnswer(sql("select 10.300000000000000000BD / 3.00000000000000000BD"), + checkAnswer(sql("select 10.3 / 3.0"), Row(BigDecimal("3.433333"))) + checkAnswer(sql("select 10.3000 / 3.0"), Row(BigDecimal("3.4333333"))) + checkAnswer(sql("select 10.30000 / 30.0"), Row(BigDecimal("0.343333333"))) + checkAnswer(sql("select 10.300000000000000000 / 3.00000000000000000"), Row(BigDecimal("3.433333333333333333333333333", new MathContext(38)))) - checkAnswer(sql("select 10.3000000000000000000BD / 3.00000000000000000BD"), + checkAnswer(sql("select 10.3000000000000000000 / 3.00000000000000000"), Row(BigDecimal("3.4333333333333333333333333333", new MathContext(38)))) } @@ -1656,13 +1656,13 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("precision smaller than scale") { - checkAnswer(sql("select 10.00BD"), Row(BigDecimal("10.00"))) - checkAnswer(sql("select 1.00BD"), Row(BigDecimal("1.00"))) - checkAnswer(sql("select 0.10BD"), Row(BigDecimal("0.10"))) - checkAnswer(sql("select 0.01BD"), Row(BigDecimal("0.01"))) - checkAnswer(sql("select 0.001BD"), Row(BigDecimal("0.001"))) - checkAnswer(sql("select -0.01BD"), Row(BigDecimal("-0.01"))) - checkAnswer(sql("select -0.001BD"), Row(BigDecimal("-0.001"))) + checkAnswer(sql("select 10.00"), Row(BigDecimal("10.00"))) + checkAnswer(sql("select 1.00"), Row(BigDecimal("1.00"))) + checkAnswer(sql("select 0.10"), Row(BigDecimal("0.10"))) + checkAnswer(sql("select 0.01"), Row(BigDecimal("0.01"))) + checkAnswer(sql("select 0.001"), Row(BigDecimal("0.001"))) + checkAnswer(sql("select -0.01"), Row(BigDecimal("-0.01"))) + checkAnswer(sql("select -0.001"), Row(BigDecimal("-0.001"))) } test("external sorting updates peak execution memory") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 8de8ba355e7d4..a3c6a1d7b20ed 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -442,13 +442,13 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { // Number and String conflict: resolve the type as number in this query. checkAnswer( - sql("select num_str + 1.2BD from jsonTable where num_str > 14"), + sql("select num_str + 1.2 from jsonTable where num_str > 14"), Row(BigDecimal("92233720368547758071.2")) ) // Number and String conflict: resolve the type as number in this query. checkAnswer( - sql("select num_str + 1.2BD from jsonTable where num_str >= 92233720368547758060BD"), + sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"), Row(new java.math.BigDecimal("92233720368547758071.2")) ) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index 828ec9710550c..554d47d651aef 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -323,7 +323,14 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { // Feature removed in HIVE-11145 "alter_partition_protect_mode", "drop_partitions_ignore_protection", - "protectmode" + "protectmode", + + // Spark parser treats numerical literals differently: it creates decimals instead of doubles. + "udf_abs", + "udf_format_number", + "udf_round", + "udf_round_3", + "view_cast" ) /** @@ -884,7 +891,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "udf_10_trims", "udf_E", "udf_PI", - "udf_abs", "udf_acos", "udf_add", "udf_array", @@ -928,7 +934,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "udf_find_in_set", "udf_float", "udf_floor", - "udf_format_number", "udf_from_unixtime", "udf_greaterthan", "udf_greaterthanorequal", @@ -976,8 +981,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "udf_regexp_replace", "udf_repeat", "udf_rlike", - "udf_round", - "udf_round_3", "udf_rpad", "udf_rtrim", "udf_sign", From d73632cd640438c3e2481417e4d116d5d6137cc6 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Sun, 17 Jan 2016 22:15:12 +0100 Subject: [PATCH 2/9] Replace Decimal Literal by Double Literal --- .../org/apache/spark/sql/catalyst/parser/ExpressionParser.g | 2 +- .../org/apache/spark/sql/catalyst/parser/SparkSqlLexer.g | 4 ++-- .../main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/ExpressionParser.g b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/ExpressionParser.g index 047a7e56cb577..957bb234e4901 100644 --- a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/ExpressionParser.g +++ b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/ExpressionParser.g @@ -122,7 +122,7 @@ constant | BigintLiteral | SmallintLiteral | TinyintLiteral - | DecimalLiteral + | DoubleLiteral | booleanValue ; diff --git a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlLexer.g b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlLexer.g index ee2882e51c450..e4ffc634e8bf4 100644 --- a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlLexer.g +++ b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlLexer.g @@ -418,9 +418,9 @@ TinyintLiteral (Digit)+ 'Y' ; -DecimalLiteral +DoubleLiteral : - Number 'B' 'D' + Number 'D' ; ByteLengthLiteral diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala index 48560e4e4fcb1..fe8d7e1851ec9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala @@ -786,8 +786,8 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C case ast if ast.tokenType == SparkSqlParser.BigintLiteral => Literal.create(ast.text.substring(0, ast.text.length() - 1).toLong, LongType) - case ast if ast.tokenType == SparkSqlParser.DecimalLiteral => - Literal(Decimal(ast.text.substring(0, ast.text.length() - 2))) + case ast if ast.tokenType == SparkSqlParser.DoubleLiteral => + Literal(ast.text.toDouble) case ast if ast.tokenType == SparkSqlParser.Number => val text = ast.text From 954f819c4431776bc436227d4215ece67584fe05 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Sun, 17 Jan 2016 22:16:20 +0100 Subject: [PATCH 3/9] Fix tests expecting double literals. --- .../HiveWindowFunctionQuerySuite.scala | 2 +- .../execution/HiveTypeCoercionSuite.scala | 20 ++++++++++++------- .../sql/hive/execution/HiveUDFSuite.scala | 4 ++-- .../sql/hive/execution/SQLQuerySuite.scala | 8 ++++---- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala index bad3ca6da231f..d0b4cbe401eb3 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala @@ -559,7 +559,7 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte """ |select p_mfgr,p_name, p_size, |histogram_numeric(p_retailprice, 5) over w1 as hist, - |percentile(p_partkey, 0.5) over w1 as per, + |percentile(p_partkey, cast(0.5 as double)) over w1 as per, |row_number() over(distribute by p_mfgr sort by p_name) as rn |from part |window w1 as (distribute by p_mfgr sort by p_mfgr, p_name diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala index d2f91861ff73b..6b424d73430e2 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala @@ -25,20 +25,26 @@ import org.apache.spark.sql.hive.test.TestHive * A set of tests that validate type promotion and coercion rules. */ class HiveTypeCoercionSuite extends HiveComparisonTest { - val baseTypes = Seq("1", "1.0", "1L", "1S", "1Y", "'1'") + val baseTypes = Seq( + ("1", "1"), + ("1.0", "CAST(1.0 AS DOUBLE)"), + ("1L", "1L"), + ("1S", "1S"), + ("1Y", "1Y"), + ("'1'", "'1'")) - baseTypes.foreach { i => - baseTypes.foreach { j => - createQueryTest(s"$i + $j", s"SELECT $i + $j FROM src LIMIT 1") + baseTypes.foreach { case (ni, si) => + baseTypes.foreach { case (nj, sj) => + createQueryTest(s"$ni + $nj", s"SELECT $si + $sj FROM src LIMIT 1") } } val nullVal = "null" - baseTypes.init.foreach { i => + baseTypes.init.foreach { case (i, s) => createQueryTest(s"case when then $i else $nullVal end ", - s"SELECT case when true then $i else $nullVal end FROM src limit 1") + s"SELECT case when true then $s else $nullVal end FROM src limit 1") createQueryTest(s"case when then $nullVal else $i end ", - s"SELECT case when true then $nullVal else $i end FROM src limit 1") + s"SELECT case when true then $nullVal else $s end FROM src limit 1") } test("[SPARK-2210] boolean cast on boolean value should be removed") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index dfe33ba8b0502..137b5bd7d74b0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -143,10 +143,10 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { } test("Generic UDAF aggregates") { - checkAnswer(sql("SELECT ceiling(percentile_approx(key, 0.99999)) FROM src LIMIT 1"), + checkAnswer(sql("SELECT ceiling(percentile_approx(key, 0.99999D)) FROM src LIMIT 1"), sql("SELECT max(key) FROM src LIMIT 1").collect().toSeq) - checkAnswer(sql("SELECT percentile_approx(100.0, array(0.9, 0.9)) FROM src LIMIT 1"), + checkAnswer(sql("SELECT percentile_approx(100.0D, array(0.9D, 0.9D)) FROM src LIMIT 1"), sql("SELECT array(100, 100) FROM src LIMIT 1").collect().toSeq) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 61d5aa7ae6b31..9cac49c57a652 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -1051,9 +1051,9 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { | java_method("java.lang.String", "isEmpty"), | java_method("java.lang.Math", "max", 2, 3), | java_method("java.lang.Math", "min", 2, 3), - | java_method("java.lang.Math", "round", 2.5), - | java_method("java.lang.Math", "exp", 1.0), - | java_method("java.lang.Math", "floor", 1.9) + | java_method("java.lang.Math", "round", 2.5D), + | java_method("java.lang.Math", "exp", 1.0D), + | java_method("java.lang.Math", "floor", 1.9D) |FROM src tablesample (1 rows) """.stripMargin), Row( @@ -1500,6 +1500,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { """ |SELECT json_tuple(json, 'f1', 'f2'), 3.14, str |FROM (SELECT '{"f1": "value1", "f2": 12}' json, 'hello' as str) test - """.stripMargin), Row("value1", "12", 3.14, "hello")) + """.stripMargin), Row("value1", "12", BigDecimal("3.14"), "hello")) } } From 0f5f6e0c7edc96b8b58d4f83f02e97b0b9e618d1 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Sun, 17 Jan 2016 22:27:06 +0100 Subject: [PATCH 4/9] Update golden files. --- ...6460d399c2d0e => '1' + 1.0-0-404b0ea20c125c9648b7919a8f41add3} | 0 ...92423c27db767ca6 => 1 + 1.0-0-77ca48f121bd2ef41efb9ee3bc28418} | 0 ...b829d43210e73 => 1.0 + '1'-0-6beb1ef5178117a9fd641008ed5ebb80} | 0 ...cd0532367bebc43 => 1.0 + 1-0-bec2842d2b009973b4d4b8f10b5554f8} | 0 ...b631d0e4f4d8d => 1.0 + 1.0-0-eafdfdbb14980ee517c388dc117d91a8} | 0 ...119a3ab1bb4205 => 1.0 + 1L-0-ef273f05968cd0e91af8c76949c73798} | 0 ...c1689680398368 => 1.0 + 1S-0-9f93538c38920d52b322bfc40cc2f31a} | 0 ...36ea612b418d43 => 1.0 + 1Y-0-9e354e022b1b423f366bf79ed7522f2a} | 0 ...51981a66cef5c8 => 1L + 1.0-0-9b0510d0bb3e9ee6a7698369b008a280} | 0 ...a2e40cbfe5349e => 1S + 1.0-0-c3d54e5b6034b7796ed16896a434d1ba} | 0 ...33231e7c2a6c8d => 1Y + 1.0-0-7b54e1d367c2ed1f5c181298ee5470d0} | 0 ...en then 1.0 else null end -0-cf71a4c4cce08635cc80a64a1ae6bc83} | 0 ...hen then null else 1.0 end -0-dfc876530eeaa7c42978d1bc0b1fd58} | 0 ...ndowing.q -- 21. testDISTs-0-d9065e533430691d70b3370174fbbd50} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename sql/hive/src/test/resources/golden/{'1' + 1.0-0-5db3b55120a19863d96460d399c2d0e => '1' + 1.0-0-404b0ea20c125c9648b7919a8f41add3} (100%) rename sql/hive/src/test/resources/golden/{1 + 1.0-0-4f5da98a11db8e7192423c27db767ca6 => 1 + 1.0-0-77ca48f121bd2ef41efb9ee3bc28418} (100%) rename sql/hive/src/test/resources/golden/{1.0 + '1'-0-a6ec78b3b93d52034aab829d43210e73 => 1.0 + '1'-0-6beb1ef5178117a9fd641008ed5ebb80} (100%) rename sql/hive/src/test/resources/golden/{1.0 + 1-0-30a4b1c8227906931cd0532367bebc43 => 1.0 + 1-0-bec2842d2b009973b4d4b8f10b5554f8} (100%) rename sql/hive/src/test/resources/golden/{1.0 + 1.0-0-87321b2e30ee2986b00b631d0e4f4d8d => 1.0 + 1.0-0-eafdfdbb14980ee517c388dc117d91a8} (100%) rename sql/hive/src/test/resources/golden/{1.0 + 1L-0-44bb88a1c9280952e8119a3ab1bb4205 => 1.0 + 1L-0-ef273f05968cd0e91af8c76949c73798} (100%) rename sql/hive/src/test/resources/golden/{1.0 + 1S-0-31fbe14d01fb532176c1689680398368 => 1.0 + 1S-0-9f93538c38920d52b322bfc40cc2f31a} (100%) rename sql/hive/src/test/resources/golden/{1.0 + 1Y-0-12bcf6e49e83abd2aa36ea612b418d43 => 1.0 + 1Y-0-9e354e022b1b423f366bf79ed7522f2a} (100%) rename sql/hive/src/test/resources/golden/{1L + 1.0-0-95a30c4b746f520f1251981a66cef5c8 => 1L + 1.0-0-9b0510d0bb3e9ee6a7698369b008a280} (100%) rename sql/hive/src/test/resources/golden/{1S + 1.0-0-8dfa46ec33c1be5ffba2e40cbfe5349e => 1S + 1.0-0-c3d54e5b6034b7796ed16896a434d1ba} (100%) rename sql/hive/src/test/resources/golden/{1Y + 1.0-0-3ad5e3db0d0300312d33231e7c2a6c8d => 1Y + 1.0-0-7b54e1d367c2ed1f5c181298ee5470d0} (100%) rename sql/hive/src/test/resources/golden/{case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0 => case when then 1.0 else null end -0-cf71a4c4cce08635cc80a64a1ae6bc83} (100%) rename sql/hive/src/test/resources/golden/{case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623 => case when then null else 1.0 end -0-dfc876530eeaa7c42978d1bc0b1fd58} (100%) rename sql/hive/src/test/resources/golden/{windowing.q -- 21. testDISTs-0-672d4cb385b7ced2e446f132474293ad => windowing.q -- 21. testDISTs-0-d9065e533430691d70b3370174fbbd50} (100%) diff --git a/sql/hive/src/test/resources/golden/'1' + 1.0-0-5db3b55120a19863d96460d399c2d0e b/sql/hive/src/test/resources/golden/'1' + 1.0-0-404b0ea20c125c9648b7919a8f41add3 similarity index 100% rename from sql/hive/src/test/resources/golden/'1' + 1.0-0-5db3b55120a19863d96460d399c2d0e rename to sql/hive/src/test/resources/golden/'1' + 1.0-0-404b0ea20c125c9648b7919a8f41add3 diff --git a/sql/hive/src/test/resources/golden/1 + 1.0-0-4f5da98a11db8e7192423c27db767ca6 b/sql/hive/src/test/resources/golden/1 + 1.0-0-77ca48f121bd2ef41efb9ee3bc28418 similarity index 100% rename from sql/hive/src/test/resources/golden/1 + 1.0-0-4f5da98a11db8e7192423c27db767ca6 rename to sql/hive/src/test/resources/golden/1 + 1.0-0-77ca48f121bd2ef41efb9ee3bc28418 diff --git a/sql/hive/src/test/resources/golden/1.0 + '1'-0-a6ec78b3b93d52034aab829d43210e73 b/sql/hive/src/test/resources/golden/1.0 + '1'-0-6beb1ef5178117a9fd641008ed5ebb80 similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + '1'-0-a6ec78b3b93d52034aab829d43210e73 rename to sql/hive/src/test/resources/golden/1.0 + '1'-0-6beb1ef5178117a9fd641008ed5ebb80 diff --git a/sql/hive/src/test/resources/golden/1.0 + 1-0-30a4b1c8227906931cd0532367bebc43 b/sql/hive/src/test/resources/golden/1.0 + 1-0-bec2842d2b009973b4d4b8f10b5554f8 similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + 1-0-30a4b1c8227906931cd0532367bebc43 rename to sql/hive/src/test/resources/golden/1.0 + 1-0-bec2842d2b009973b4d4b8f10b5554f8 diff --git a/sql/hive/src/test/resources/golden/1.0 + 1.0-0-87321b2e30ee2986b00b631d0e4f4d8d b/sql/hive/src/test/resources/golden/1.0 + 1.0-0-eafdfdbb14980ee517c388dc117d91a8 similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + 1.0-0-87321b2e30ee2986b00b631d0e4f4d8d rename to sql/hive/src/test/resources/golden/1.0 + 1.0-0-eafdfdbb14980ee517c388dc117d91a8 diff --git a/sql/hive/src/test/resources/golden/1.0 + 1L-0-44bb88a1c9280952e8119a3ab1bb4205 b/sql/hive/src/test/resources/golden/1.0 + 1L-0-ef273f05968cd0e91af8c76949c73798 similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + 1L-0-44bb88a1c9280952e8119a3ab1bb4205 rename to sql/hive/src/test/resources/golden/1.0 + 1L-0-ef273f05968cd0e91af8c76949c73798 diff --git a/sql/hive/src/test/resources/golden/1.0 + 1S-0-31fbe14d01fb532176c1689680398368 b/sql/hive/src/test/resources/golden/1.0 + 1S-0-9f93538c38920d52b322bfc40cc2f31a similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + 1S-0-31fbe14d01fb532176c1689680398368 rename to sql/hive/src/test/resources/golden/1.0 + 1S-0-9f93538c38920d52b322bfc40cc2f31a diff --git a/sql/hive/src/test/resources/golden/1.0 + 1Y-0-12bcf6e49e83abd2aa36ea612b418d43 b/sql/hive/src/test/resources/golden/1.0 + 1Y-0-9e354e022b1b423f366bf79ed7522f2a similarity index 100% rename from sql/hive/src/test/resources/golden/1.0 + 1Y-0-12bcf6e49e83abd2aa36ea612b418d43 rename to sql/hive/src/test/resources/golden/1.0 + 1Y-0-9e354e022b1b423f366bf79ed7522f2a diff --git a/sql/hive/src/test/resources/golden/1L + 1.0-0-95a30c4b746f520f1251981a66cef5c8 b/sql/hive/src/test/resources/golden/1L + 1.0-0-9b0510d0bb3e9ee6a7698369b008a280 similarity index 100% rename from sql/hive/src/test/resources/golden/1L + 1.0-0-95a30c4b746f520f1251981a66cef5c8 rename to sql/hive/src/test/resources/golden/1L + 1.0-0-9b0510d0bb3e9ee6a7698369b008a280 diff --git a/sql/hive/src/test/resources/golden/1S + 1.0-0-8dfa46ec33c1be5ffba2e40cbfe5349e b/sql/hive/src/test/resources/golden/1S + 1.0-0-c3d54e5b6034b7796ed16896a434d1ba similarity index 100% rename from sql/hive/src/test/resources/golden/1S + 1.0-0-8dfa46ec33c1be5ffba2e40cbfe5349e rename to sql/hive/src/test/resources/golden/1S + 1.0-0-c3d54e5b6034b7796ed16896a434d1ba diff --git a/sql/hive/src/test/resources/golden/1Y + 1.0-0-3ad5e3db0d0300312d33231e7c2a6c8d b/sql/hive/src/test/resources/golden/1Y + 1.0-0-7b54e1d367c2ed1f5c181298ee5470d0 similarity index 100% rename from sql/hive/src/test/resources/golden/1Y + 1.0-0-3ad5e3db0d0300312d33231e7c2a6c8d rename to sql/hive/src/test/resources/golden/1Y + 1.0-0-7b54e1d367c2ed1f5c181298ee5470d0 diff --git a/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0 b/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-cf71a4c4cce08635cc80a64a1ae6bc83 similarity index 100% rename from sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0 rename to sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-cf71a4c4cce08635cc80a64a1ae6bc83 diff --git a/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623 b/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-dfc876530eeaa7c42978d1bc0b1fd58 similarity index 100% rename from sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623 rename to sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-dfc876530eeaa7c42978d1bc0b1fd58 diff --git a/sql/hive/src/test/resources/golden/windowing.q -- 21. testDISTs-0-672d4cb385b7ced2e446f132474293ad b/sql/hive/src/test/resources/golden/windowing.q -- 21. testDISTs-0-d9065e533430691d70b3370174fbbd50 similarity index 100% rename from sql/hive/src/test/resources/golden/windowing.q -- 21. testDISTs-0-672d4cb385b7ced2e446f132474293ad rename to sql/hive/src/test/resources/golden/windowing.q -- 21. testDISTs-0-d9065e533430691d70b3370174fbbd50 From 5c72b58fd5b323a73991d45ad7facc9b23ce6ddd Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Sun, 17 Jan 2016 22:31:00 +0100 Subject: [PATCH 5/9] Fix R result. --- R/pkg/inst/tests/testthat/test_sparkSQL.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R index 67ecdbc522d23..27ad9f3958362 100644 --- a/R/pkg/inst/tests/testthat/test_sparkSQL.R +++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R @@ -1781,7 +1781,7 @@ test_that("Method coltypes() to get and set R's data types of a DataFrame", { expect_equal(coltypes(x), "map") df <- selectExpr(read.json(sqlContext, jsonPath), "name", "(age * 1.21) as age") - expect_equal(dtypes(df), list(c("name", "string"), c("age", "double"))) + expect_equal(dtypes(df), list(c("name", "string"), c("age", "decimal(24,2)"))) df1 <- select(df, cast(df$age, "integer")) coltypes(df) <- c("character", "integer") From 0ac091a89b674c75b3e622e9a626c61d7dd719c4 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Mon, 18 Jan 2016 22:45:31 +0100 Subject: [PATCH 6/9] Add debug code to BucketedWriteSuite (revert later on) --- .../spark/sql/sources/BucketedWriteSuite.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala index e812439bed9aa..f8096146448ce 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala @@ -70,7 +70,19 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle source: String, bucketCols: Seq[String], sortCols: Seq[String] = Nil): Unit = { - val allBucketFiles = dataDir.listFiles().filterNot(f => + val files = dataDir.listFiles() + if (files == null) { + dataDir.isDirectory + throw new NullPointerException(s""" + |dataDir.listFiles() is NULL + |== Directory(isDirectory=${dataDir.isDirectory}) == + |$dataDir + |== Hive Warehouse(isDirectory=${hiveContext.warehousePath.isDirectory}) == + |${hiveContext.warehousePath} + |== bucketed_table TableIdentifier == + |${sqlContext.sqlParser.parseTableIdentifier("bucketed_table")}""".stripMargin) + } + val allBucketFiles = files.filterNot(f => f.getName.startsWith(".") || f.getName.startsWith("_") ) val groupedBucketFiles = allBucketFiles.groupBy(f => BucketingUtils.getBucketId(f.getName).get) From 1b0508f6900b16db2659ac6c62f123a6587a5386 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Tue, 19 Jan 2016 23:12:14 +0100 Subject: [PATCH 7/9] Fix naming mismatch. --- .../spark/sql/sources/BucketedWriteSuite.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala index e374114f656bc..55a2dd952e057 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.sources import java.io.File +import java.net.URI import org.apache.spark.sql.{AnalysisException, QueryTest} import org.apache.spark.sql.catalyst.expressions.UnsafeProjection @@ -65,6 +66,11 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle private val df = (0 until 50).map(i => (i % 5, i % 13, i.toString)).toDF("i", "j", "k") + def tableDir: File = { + val identifier = hiveContext.sqlParser.parseTableIdentifier("bucketed_table") + new File(URI.create(hiveContext.catalog.hiveDefaultTableFilePath(identifier))) + } + /** * A helper method to check the bucket write functionality in low level, i.e. check the written * bucket files to see if the data are correct. User should pass in a data dir that these bucket @@ -82,7 +88,7 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle dataDir.isDirectory throw new NullPointerException(s""" |dataDir.listFiles() is NULL - |== Directory(isDirectory=${dataDir.isDirectory}) == + |== Directory(isDirectory=${dataDir.isDirectory} exists=${dataDir.exists}) == |$dataDir |== Hive Warehouse(isDirectory=${hiveContext.warehousePath.isDirectory}) == |${hiveContext.warehousePath} @@ -139,7 +145,6 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle .bucketBy(8, "j", "k") .saveAsTable("bucketed_table") - val tableDir = new File(hiveContext.warehousePath, "bucketed_table") for (i <- 0 until 5) { testBucketing(new File(tableDir, s"i=$i"), source, 8, Seq("j", "k")) } @@ -157,7 +162,6 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle .sortBy("k") .saveAsTable("bucketed_table") - val tableDir = new File(hiveContext.warehousePath, "bucketed_table") for (i <- 0 until 5) { testBucketing(new File(tableDir, s"i=$i"), source, 8, Seq("j"), Seq("k")) } @@ -173,7 +177,6 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle .bucketBy(8, "i", "j") .saveAsTable("bucketed_table") - val tableDir = new File(hiveContext.warehousePath, "bucketed_table") testBucketing(tableDir, source, 8, Seq("i", "j")) } } @@ -188,7 +191,6 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle .sortBy("k") .saveAsTable("bucketed_table") - val tableDir = new File(hiveContext.warehousePath, "bucketed_table") testBucketing(tableDir, source, 8, Seq("i", "j"), Seq("k")) } } From 8bf2a278915b9bb35424cfd91d2130018af973ed Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Wed, 20 Jan 2016 08:31:11 +0100 Subject: [PATCH 8/9] Add checks no fix yet... --- .../org/apache/spark/sql/hive/execution/HiveExplainSuite.scala | 1 + .../src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala index b7ef5d1db7297..d7727663fcc6e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala @@ -41,6 +41,7 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto } test("explain create table command") { + sql("show tables").show() checkExistence(sql("explain create table temp__b as select * from src limit 2"), true, "== Physical Plan ==", "InsertIntoHiveTable", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala index c997453803b09..2826e9dc4f826 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala @@ -452,6 +452,7 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { """.stripMargin) var tableIdentifier = catalog.QualifiedTableName("default", "test_insert_parquet") + assert(catalog.client.currentDatabase === "default", "Current database should be 'default'.") // First, make sure the converted test_parquet is not cached. assert(catalog.cachedDataSourceTables.getIfPresent(tableIdentifier) === null) From 9afd729117ec9a2632d0c219b8ffa329e4b35943 Mon Sep 17 00:00:00 2001 From: Herman van Hovell Date: Wed, 20 Jan 2016 13:51:08 +0100 Subject: [PATCH 9/9] Revert Debugging code. --- .../sql/hive/execution/HiveExplainSuite.scala | 1 - .../org/apache/spark/sql/hive/parquetSuites.scala | 1 - .../spark/sql/sources/BucketedWriteSuite.scala | 14 +------------- 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala index d7727663fcc6e..b7ef5d1db7297 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala @@ -41,7 +41,6 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto } test("explain create table command") { - sql("show tables").show() checkExistence(sql("explain create table temp__b as select * from src limit 2"), true, "== Physical Plan ==", "InsertIntoHiveTable", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala index 2826e9dc4f826..c997453803b09 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala @@ -452,7 +452,6 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest { """.stripMargin) var tableIdentifier = catalog.QualifiedTableName("default", "test_insert_parquet") - assert(catalog.client.currentDatabase === "default", "Current database should be 'default'.") // First, make sure the converted test_parquet is not cached. assert(catalog.cachedDataSourceTables.getIfPresent(tableIdentifier) === null) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala index 55a2dd952e057..8cac7fe48f171 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala @@ -83,19 +83,7 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle numBuckets: Int, bucketCols: Seq[String], sortCols: Seq[String] = Nil): Unit = { - val files = dataDir.listFiles() - if (files == null) { - dataDir.isDirectory - throw new NullPointerException(s""" - |dataDir.listFiles() is NULL - |== Directory(isDirectory=${dataDir.isDirectory} exists=${dataDir.exists}) == - |$dataDir - |== Hive Warehouse(isDirectory=${hiveContext.warehousePath.isDirectory}) == - |${hiveContext.warehousePath} - |== bucketed_table TableIdentifier == - |${sqlContext.sqlParser.parseTableIdentifier("bucketed_table")}""".stripMargin) - } - val allBucketFiles = files.filterNot(f => + val allBucketFiles = dataDir.listFiles().filterNot(f => f.getName.startsWith(".") || f.getName.startsWith("_") )