From 3242532d4815fa9595dd9ca2d4d0b86c6d206ddb Mon Sep 17 00:00:00 2001 From: Devaraj K Date: Fri, 21 Jul 2017 14:51:46 -0700 Subject: [PATCH 1/2] [SPARK-21339] [CORE] spark-shell --packages option does not add jars to classpath on windows --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 3 +-- .../test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 0ea14361b2f77..c70ac920e10cb 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -649,7 +649,6 @@ object SparkSubmit extends CommandLineUtils { // Resolve paths in certain spark properties val pathConfigs = Seq( - "spark.jars", "spark.files", "spark.yarn.dist.files", "spark.yarn.dist.archives", @@ -973,7 +972,7 @@ object SparkSubmit extends CommandLineUtils { paths.split(",").map(_.trim).filter(_.nonEmpty).flatMap { path => val uri = Utils.resolveURI(path) uri.getScheme match { - case "local" | "http" | "https" | "ftp" => Array(path) + case "local" | "file" | "http" | "https" | "ftp" => Array(path) case _ => val fs = FileSystem.get(uri, hadoopConf) Option(fs.globStatus(new Path(uri))).map { status => diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 97357cdbb6083..44891ac23d954 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -631,7 +631,7 @@ class SparkSubmitSuite ) val appArgs = new SparkSubmitArguments(clArgs) val sysProps = SparkSubmit.prepareSubmitEnvironment(appArgs)._3 - sysProps("spark.jars") should be(Utils.resolveURIs(jars + ",thejar.jar")) + sysProps("spark.jars") should be(jars + ',' + Utils.resolveURIs("thejar.jar")) sysProps("spark.files") should be(Utils.resolveURIs(files)) // Test files and archives (Yarn) From 025e484f5fd768974910e24357d4d56b61b801e7 Mon Sep 17 00:00:00 2001 From: Devaraj K Date: Tue, 25 Jul 2017 20:27:53 -0700 Subject: [PATCH 2/2] Removing the file:/ scheme from each jar before setting into classpath --- .../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 3 ++- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +- .../src/main/scala/org/apache/spark/repl/Main.scala | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index c70ac920e10cb..0ea14361b2f77 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -649,6 +649,7 @@ object SparkSubmit extends CommandLineUtils { // Resolve paths in certain spark properties val pathConfigs = Seq( + "spark.jars", "spark.files", "spark.yarn.dist.files", "spark.yarn.dist.archives", @@ -972,7 +973,7 @@ object SparkSubmit extends CommandLineUtils { paths.split(",").map(_.trim).filter(_.nonEmpty).flatMap { path => val uri = Utils.resolveURI(path) uri.getScheme match { - case "local" | "file" | "http" | "https" | "ftp" => Array(path) + case "local" | "http" | "https" | "ftp" => Array(path) case _ => val fs = FileSystem.get(uri, hadoopConf) Option(fs.globStatus(new Path(uri))).map { status => diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 44891ac23d954..97357cdbb6083 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -631,7 +631,7 @@ class SparkSubmitSuite ) val appArgs = new SparkSubmitArguments(clArgs) val sysProps = SparkSubmit.prepareSubmitEnvironment(appArgs)._3 - sysProps("spark.jars") should be(jars + ',' + Utils.resolveURIs("thejar.jar")) + sysProps("spark.jars") should be(Utils.resolveURIs(jars + ",thejar.jar")) sysProps("spark.files") should be(Utils.resolveURIs(files)) // Test files and archives (Yarn) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala index b8b38e828b255..a2e50b4b2a2f8 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala @@ -18,6 +18,7 @@ package org.apache.spark.repl import java.io.File +import java.net.URI import java.util.Locale import scala.tools.nsc.GenericRunnerSettings @@ -56,7 +57,10 @@ object Main extends Logging { // Visible for testing private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = { interp = _interp - val jars = Utils.getUserJars(conf, isShell = true).mkString(File.pathSeparator) + val jars = Utils.getUserJars(conf, isShell = true) + // Remove file:///, file:// or file:/ scheme if exists for each jar + .map(x => if (x.startsWith("file:")) new File(new URI(x)).getPath else x) + .mkString(File.pathSeparator) val interpArguments = List( "-Yrepl-class-based", "-Yrepl-outdir", s"${outputDir.getAbsolutePath}",