From 19c6a51a7e54a18325c76d661dc581565eed9441 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 12 Feb 2020 23:07:27 +0300 Subject: [PATCH 1/9] Enable Java 8 time API in thrift server --- .../org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala index 8944b93d9b697..233e6224a10d9 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala @@ -24,6 +24,7 @@ import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.internal.Logging import org.apache.spark.sql.{SparkSession, SQLContext} import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.Utils /** A singleton object for the master program. The slaves should not access this. */ @@ -45,6 +46,8 @@ private[hive] object SparkSQLEnv extends Logging { sparkConf .setAppName(maybeAppName.getOrElse(s"SparkSQL::${Utils.localHostName()}")) + .set(SQLConf.DATETIME_JAVA8API_ENABLED, true) + val sparkSession = SparkSession.builder.config(sparkConf).enableHiveSupport().getOrCreate() sparkContext = sparkSession.sparkContext From 930c2935b6b493aa9dcfa26fb85112691283ae62 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Thu, 4 Jun 2020 19:42:40 +0300 Subject: [PATCH 2/9] Set Java 8 Api in SparkOperation --- .../org/apache/spark/sql/hive/thriftserver/SparkOperation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala index 3da568cfa256e..fe465fcf9d36a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala @@ -73,7 +73,7 @@ private[hive] trait SparkOperation extends Operation with Logging { sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool) case None => } - + sqlContext.sparkContext.setLocalProperty(SQLConf.DATETIME_JAVA8API_ENABLED.key, "true") // run the body f } finally { From 51a377784dc0459ba9006e2571c7865b1cd3ce83 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Thu, 4 Jun 2020 22:43:55 +0300 Subject: [PATCH 3/9] Revert "Set Java 8 Api in SparkOperation" This reverts commit 930c2935b6b493aa9dcfa26fb85112691283ae62. --- .../org/apache/spark/sql/hive/thriftserver/SparkOperation.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala index fe465fcf9d36a..3da568cfa256e 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala @@ -73,7 +73,7 @@ private[hive] trait SparkOperation extends Operation with Logging { sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool) case None => } - sqlContext.sparkContext.setLocalProperty(SQLConf.DATETIME_JAVA8API_ENABLED.key, "true") + // run the body f } finally { From 6f132a38239eb27ebfbac9c160534858fef89d1d Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Thu, 4 Jun 2020 22:44:41 +0300 Subject: [PATCH 4/9] Revert "Enable Java 8 time API in thrift server" This reverts commit 19c6a51a7e54a18325c76d661dc581565eed9441. --- .../org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala index 233e6224a10d9..8944b93d9b697 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala @@ -24,7 +24,6 @@ import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.internal.Logging import org.apache.spark.sql.{SparkSession, SQLContext} import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.Utils /** A singleton object for the master program. The slaves should not access this. */ @@ -46,8 +45,6 @@ private[hive] object SparkSQLEnv extends Logging { sparkConf .setAppName(maybeAppName.getOrElse(s"SparkSQL::${Utils.localHostName()}")) - .set(SQLConf.DATETIME_JAVA8API_ENABLED, true) - val sparkSession = SparkSession.builder.config(sparkConf).enableHiveSupport().getOrCreate() sparkContext = sparkSession.sparkContext From 252219fc0299a4a1f34bdfbb637ca126cc3a2c65 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Thu, 4 Jun 2020 23:28:50 +0300 Subject: [PATCH 5/9] Add test --- .../hive/thriftserver/HiveThriftServer2Suites.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 21256ad02c134..e897ccd3e926e 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -866,6 +866,18 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } } } + + test("SPARK-30808: use Java 8 time API and Proleptic Gregorian calendar by default") { + withJdbcStatement() { st => + // Proleptic Gregorian calendar has no gap in the range 1582-10-04..1582-10-15 + val date = "1582-10-10" + val rs = st.executeQuery(s"select date '$date'") + rs.next() + assert(rs.getString(1) === date) + // 1582-10-10 is shifted to the next valid date in the hybrid calendar by java.sql.Date + assert(rs.getDate(1).toString() === "1582-10-15") + } + } } class SingleSessionSuite extends HiveThriftJdbcTest { From 34f7db3b5dd96ce0b6ed261e934f28aa19cbc38e Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Fri, 5 Jun 2020 00:03:11 +0300 Subject: [PATCH 6/9] Set Java 8 Api in SparkSQLSessionManager --- .../spark/sql/hive/thriftserver/SparkSQLSessionManager.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala index e10e7ed1a2769..806b6146b2db1 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.SQLContext import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.hive.thriftserver.ReflectionUtils._ import org.apache.spark.sql.hive.thriftserver.server.SparkSQLOperationManager +import org.apache.spark.sql.internal.SQLConf private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext: SQLContext) @@ -63,6 +64,7 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext: sqlContext.newSession() } ctx.setConf(HiveUtils.FAKE_HIVE_VERSION.key, HiveUtils.builtinHiveVersion) + ctx.setConf(SQLConf.DATETIME_JAVA8API_ENABLED, true) val hiveSessionState = session.getSessionState setConfMap(ctx, hiveSessionState.getOverriddenConfigurations) setConfMap(ctx, hiveSessionState.getHiveVariables) From 76b4ccddef7194eeca3c0e6505a5062678113608 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Fri, 5 Jun 2020 00:03:24 +0300 Subject: [PATCH 7/9] Fix test --- .../sql/hive/thriftserver/HiveThriftServer2Suites.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index e897ccd3e926e..835aee0a598bf 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -873,9 +873,9 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { val date = "1582-10-10" val rs = st.executeQuery(s"select date '$date'") rs.next() - assert(rs.getString(1) === date) - // 1582-10-10 is shifted to the next valid date in the hybrid calendar by java.sql.Date - assert(rs.getDate(1).toString() === "1582-10-15") + val expected = java.sql.Date.valueOf(date) + assert(rs.getDate(1) === expected) + assert(rs.getString(1) === expected.toString) } } } From 3c35cf5920c6e4216adcefc866bd518dfe635def Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 12 Feb 2020 23:07:27 +0300 Subject: [PATCH 8/9] Enable Java 8 time API in thrift server --- .../org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala index 8944b93d9b697..233e6224a10d9 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala @@ -24,6 +24,7 @@ import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.internal.Logging import org.apache.spark.sql.{SparkSession, SQLContext} import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.Utils /** A singleton object for the master program. The slaves should not access this. */ @@ -45,6 +46,8 @@ private[hive] object SparkSQLEnv extends Logging { sparkConf .setAppName(maybeAppName.getOrElse(s"SparkSQL::${Utils.localHostName()}")) + .set(SQLConf.DATETIME_JAVA8API_ENABLED, true) + val sparkSession = SparkSession.builder.config(sparkConf).enableHiveSupport().getOrCreate() sparkContext = sparkSession.sparkContext From f3e5357a2ca7b650593442da8ee249b928c2d944 Mon Sep 17 00:00:00 2001 From: Max Gekk Date: Fri, 5 Jun 2020 15:13:07 +0300 Subject: [PATCH 9/9] Add a test to CliSuite --- .../org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index ea1a371151c36..8546421a86927 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -551,4 +551,10 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { errorResponses = Seq("AnalysisException"))( ("", "Error in query: The second argument of 'date_sub' function needs to be an integer.")) } + + test("SPARK-30808: use Java 8 time API in Thrift SQL CLI by default") { + // If Java 8 time API is enabled via the SQL config `spark.sql.datetime.java8API.enabled`, + // the date formatter for `java.sql.LocalDate` must output negative years with sign. + runCliWithin(1.minute)("SELECT MAKE_DATE(-44, 3, 15);" -> "-0044-03-15") + } }