From ee868b91351ea86f4f756139ea93ac6fc89871ea Mon Sep 17 00:00:00 2001 From: weiwenda <994184916@qq.com> Date: Fri, 24 Jan 2020 08:58:07 +0800 Subject: [PATCH 1/2] stop check values of spark.sql.catalogImplementation to provide an approach to let user define themselves metadata catalog --- .../scala/org/apache/spark/sql/internal/StaticSQLConf.scala | 6 +++++- .../src/main/scala/org/apache/spark/sql/SparkSession.scala | 6 ++++++ .../scala/org/apache/spark/sql/internal/SharedState.scala | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index 66ac9ddb21aaa..7277ae83d3416 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -36,9 +36,13 @@ object StaticSQLConf { .createWithDefault(Utils.resolveURI("spark-warehouse").toString) val CATALOG_IMPLEMENTATION = buildStaticConf("spark.sql.catalogImplementation") + .doc("Spark built-in implementation is in-memory and hive," + + "when set to yourImpl(can be any words), " + + "you need also provide following configuration to make everything going" + + "spark.sql.catalogImplementation.yourImpl.builder" + + "spark.sql.catalogImplementation.yourImpl.externalCatalog") .internal() .stringConf - .checkValues(Set("hive", "in-memory")) .createWithDefault("in-memory") val GLOBAL_TEMP_DATABASE = buildStaticConf("spark.sql.globalTempDatabase") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index bd2bc1c0ad5d7..0d01e0d6cce25 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -1026,6 +1026,12 @@ object SparkSession extends Logging { conf.get(CATALOG_IMPLEMENTATION) match { case "hive" => HIVE_SESSION_STATE_BUILDER_CLASS_NAME case "in-memory" => classOf[SessionStateBuilder].getCanonicalName + case other => conf.getOption(s"spark.sql.catalogImplementation.$other.builder") + .getOrElse { + throw new IllegalArgumentException( + "You need to configure spark.sql.catalogImplementation.xx.builder when xx configured by" + + "spark.sql.catalogImplementation is not in-memory nor hive") + } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index de3805e105802..250513d42f4d9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -217,6 +217,12 @@ object SharedState extends Logging { conf.get(CATALOG_IMPLEMENTATION) match { case "hive" => HIVE_EXTERNAL_CATALOG_CLASS_NAME case "in-memory" => classOf[InMemoryCatalog].getCanonicalName + case other => conf.getOption(s"spark.sql.catalogImplementation.$other.externalCatalog") + .getOrElse { + throw new IllegalArgumentException( + "You need to configure spark.sql.catalogImplementation.xx.externalCatalog when xx " + + "configured by spark.sql.catalogImplementation is not in-memory nor hive") + } } } From e01b3520542909ddcc4cd0c06a86c293bfc4adb2 Mon Sep 17 00:00:00 2001 From: weiwenda <994184916@qq.com> Date: Fri, 24 Jan 2020 17:34:31 +0800 Subject: [PATCH 2/2] code reformat --- .../org/apache/spark/sql/internal/StaticSQLConf.scala | 9 ++++----- .../main/scala/org/apache/spark/sql/SparkSession.scala | 4 ++-- .../org/apache/spark/sql/internal/SharedState.scala | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala index 7277ae83d3416..3f10e2d88afe9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala @@ -36,12 +36,11 @@ object StaticSQLConf { .createWithDefault(Utils.resolveURI("spark-warehouse").toString) val CATALOG_IMPLEMENTATION = buildStaticConf("spark.sql.catalogImplementation") - .doc("Spark built-in implementation is in-memory and hive," + + .doc("Spark built-in implementation is in-memory and hive, " + "when set to yourImpl(can be any words), " + - "you need also provide following configuration to make everything going" + - "spark.sql.catalogImplementation.yourImpl.builder" + - "spark.sql.catalogImplementation.yourImpl.externalCatalog") - .internal() + "you need also provide following configurations to make everything going: " + + "`spark.sql.catalogImplementation.yourImpl.builder` " + + "`spark.sql.catalogImplementation.yourImpl.externalCatalog`.") .stringConf .createWithDefault("in-memory") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 0d01e0d6cce25..aa603f2faed27 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -1029,8 +1029,8 @@ object SparkSession extends Logging { case other => conf.getOption(s"spark.sql.catalogImplementation.$other.builder") .getOrElse { throw new IllegalArgumentException( - "You need to configure spark.sql.catalogImplementation.xx.builder when xx configured by" + - "spark.sql.catalogImplementation is not in-memory nor hive") + s"You need to configure spark.sql.catalogImplementation.$other.builder when " + + s"`$other` configured by spark.sql.catalogImplementation is not in-memory nor hive.") } } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 250513d42f4d9..08e3a0c15c686 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -220,8 +220,8 @@ object SharedState extends Logging { case other => conf.getOption(s"spark.sql.catalogImplementation.$other.externalCatalog") .getOrElse { throw new IllegalArgumentException( - "You need to configure spark.sql.catalogImplementation.xx.externalCatalog when xx " + - "configured by spark.sql.catalogImplementation is not in-memory nor hive") + s"You need to configure spark.sql.catalogImplementation.$other.externalCatalog when " + + s"`$other` configured by spark.sql.catalogImplementation is not in-memory nor hive.") } } }