From cd8e9a4ddf704f2d01df870cc898af53e62a9d2f Mon Sep 17 00:00:00 2001 From: tone-zhang Date: Thu, 8 Sep 2016 09:22:09 +0000 Subject: [PATCH] [SPARK-17330] Clean up the temporary data files used in Spark UT When run Spark UT based on the latest version of master branch, One UT case is failed becaues the temporary data files have not been cleaned up. The UT name is "SPARK-8368: includes jars passed in through --jars". In Spark UT, the temorary data file should be stored in spark/target/tmp not in spark/spark-warehouse. --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 2 ++ .../org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 0073659a31541..8dda042ecc21a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -43,6 +43,8 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { // drop all databases, tables and functions after each test spark.sessionState.catalog.reset() } finally { + val path = System.getProperty("user.dir") + "/spark-warehouse" + Utils.deleteRecursively(new File(path)) super.afterEach() } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index af282866669b8..29317e2887861 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -590,7 +590,9 @@ object SparkSubmitClassLoaderTest extends Logging { def main(args: Array[String]) { Utils.configTestLog4j("INFO") val conf = new SparkConf() + val hiveWarehouseLocation = Utils.createTempDir() conf.set("spark.ui.enabled", "false") + conf.set("spark.sql.warehouse.dir", hiveWarehouseLocation.toString) val sc = new SparkContext(conf) val hiveContext = new TestHiveContext(sc) val df = hiveContext.createDataFrame((1 to 100).map(i => (i, i))).toDF("i", "j") @@ -699,11 +701,13 @@ object SPARK_9757 extends QueryTest { def main(args: Array[String]): Unit = { Utils.configTestLog4j("INFO") + val hiveWarehouseLocation = Utils.createTempDir() val sparkContext = new SparkContext( new SparkConf() .set("spark.sql.hive.metastore.version", "0.13.1") .set("spark.sql.hive.metastore.jars", "maven") - .set("spark.ui.enabled", "false")) + .set("spark.ui.enabled", "false") + .set("spark.sql.warehouse.dir", hiveWarehouseLocation.toString)) val hiveContext = new TestHiveContext(sparkContext) spark = hiveContext.sparkSession