From 94b5ac8ee9b8679620e4b73d46f527cfd9e1bbb4 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 17 Jan 2017 13:14:38 -0800 Subject: [PATCH 1/5] Add a test cases for `SET -v`. --- .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++-- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 645b0fa13ee3f..c421819cac345 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -948,9 +948,9 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { * definition contains key, defaultValue and doc. */ def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized { - sqlConfEntries.values.asScala.filter(_.isPublic).map { entry => + sqlConfEntries.asScala.toMap.values.filter(_.isPublic).map { entry => (entry.key, getConfString(entry.key, entry.defaultValueString), entry.doc) - }.toSeq + }.toArray.toSeq } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 8f1beaa3a1685..6fc253d629523 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -992,6 +992,10 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { spark.sessionState.conf.clear() } + test("`SET -v` collect test") { + sql("SET -v").collect() + } + test("apply schema") { val schema1 = StructType( StructField("f1", IntegerType, false) :: From 047a1f21ca316ba26499606821984b52cf07ef2c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 21 Jan 2017 11:55:45 -0800 Subject: [PATCH 2/5] add show test. --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 6fc253d629523..d4748f202f32a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -992,7 +992,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { spark.sessionState.conf.clear() } - test("`SET -v` collect test") { + test("`SET -v` show") { + sql("SET -v").show(200, false) + } + + test("`SET -v` collect") { sql("SET -v").collect() } From 3513e5942339a3c202d824c412da1ba4466a38fe Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 21 Jan 2017 14:44:38 -0800 Subject: [PATCH 3/5] fix --- .../org/apache/spark/sql/execution/command/SetCommand.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala index dc8d97594c7a7..68c1d97b00bfb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala @@ -89,7 +89,7 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm val runFunc = (sparkSession: SparkSession) => { sparkSession.sessionState.conf.getAllDefinedConfs.map { case (key, defaultValue, doc) => Row(key, defaultValue, doc) - } + }.toArray.toSeq } val schema = StructType( StructField("key", StringType, nullable = false) :: From 042f0258582445b5541220eae1e3ddc4cd5ebfce Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 21 Jan 2017 15:08:26 -0800 Subject: [PATCH 4/5] Add reset to prevent other settings. --- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index d4748f202f32a..0a379ff713016 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -993,10 +993,12 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { } test("`SET -v` show") { + sql("RESET") sql("SET -v").show(200, false) } test("`SET -v` collect") { + sql("RESET") sql("SET -v").collect() } From 075f4667020438a650659197ac8212c785775e75 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 21 Jan 2017 15:53:52 -0800 Subject: [PATCH 5/5] Fix all. --- .../org/apache/spark/sql/execution/command/SetCommand.scala | 4 ++-- .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++-- .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 6 ------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala index 68c1d97b00bfb..1819d190aebd4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala @@ -88,8 +88,8 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm case Some(("-v", None)) => val runFunc = (sparkSession: SparkSession) => { sparkSession.sessionState.conf.getAllDefinedConfs.map { case (key, defaultValue, doc) => - Row(key, defaultValue, doc) - }.toArray.toSeq + Row(key, if (defaultValue == null) "null" else defaultValue, doc) + } } val schema = StructType( StructField("key", StringType, nullable = false) :: diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c421819cac345..645b0fa13ee3f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -948,9 +948,9 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { * definition contains key, defaultValue and doc. */ def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized { - sqlConfEntries.asScala.toMap.values.filter(_.isPublic).map { entry => + sqlConfEntries.values.asScala.filter(_.isPublic).map { entry => (entry.key, getConfString(entry.key, entry.defaultValueString), entry.doc) - }.toArray.toSeq + }.toSeq } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 0a379ff713016..049e6ed2b1aa2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -992,13 +992,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { spark.sessionState.conf.clear() } - test("`SET -v` show") { - sql("RESET") - sql("SET -v").show(200, false) - } - test("`SET -v` collect") { - sql("RESET") sql("SET -v").collect() }