From 42947032e565f6eccfa2ccceb26b00a3da7d9ad6 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 12:02:46 -0700 Subject: [PATCH 1/8] Remove Scala version from parent POM artifact ID --- assembly/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- dev/change-scala-version.sh | 5 ----- examples/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/java8-tests/pom.xml | 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 4 +--- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- yarn/pom.xml | 2 +- 33 files changed, 32 insertions(+), 39 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 22cbac06cad61..4d35630f30925 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index bd507c2cb6c4b..0232bbd887b10 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 810ec10ca05b3..e3315d9191316 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index bc83ef24c30ec..d5aa743f970cc 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 8bc1f52798941..231ca65f0e6bd 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 8e702b4fefe8c..de6852cf35d5e 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 93b9580f26b86..0c3cbcd6487f2 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 4c7e3a36620a9..7f825e0033ab3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh index d7975dfb6475c..ce6d32300cba4 100755 --- a/dev/change-scala-version.sh +++ b/dev/change-scala-version.sh @@ -60,11 +60,6 @@ BASEDIR=$(dirname $0)/.. find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \; -# Also update in parent POM -# Match any scala binary version to ensure idempotency -sed_i '1,/[0-9]*\.[0-9]*[0-9]*\.[0-9]*'$TO_VERSION'4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 17fd7d781c9ab..040aec1f82f98 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index ac15b93c048da..3fbc5483c7af3 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index e4effe158c826..a5fa67a36ed38 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index d650dd034d636..ac746aafd32ab 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml index 1ea9196e9dfe3..d9e3578c07435 100644 --- a/external/java8-tests/pom.xml +++ b/external/java8-tests/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 62818f5e8f434..378905e057e6d 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 68d52e9339b3d..971bd3b525535 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index d1c38c7ca5d69..4a3bde2d82e2f 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 935155eb5d362..c846dc1348141 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index bfb92791de3d8..baa8c0a4414a0 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 1813f383cdcba..47481d65d7d14 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index ef731948826ef..d428aaaacda65 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index c56561f215926..62cc1a49ce322 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index e56eafc3006bd..25f5747aed368 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 4585c8b9c2b0b..ad27d94806996 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 14 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT pom Spark Project Parent POM @@ -158,8 +158,6 @@ 3.4.1 3.2.2 - 2.11.8 - 2.11 ${scala.version} org.scala-lang 1.9.13 diff --git a/repl/pom.xml b/repl/pom.xml index 0f396c9b809bd..294f4723778e0 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 1748fa2778d6a..fd01adcb626b8 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 8b1017042cd93..0bf33af5314bf 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index c8d17bd468582..d8d3d5b484732 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 61504becf1f38..e6bc97e65656b 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 7d409c5d3b076..9ac0f90a5b93e 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 9bb20e1381067..0b9241b7f362f 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index 328bb6678db99..24d5c3e1fc5ef 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.spark - spark-parent_2.11 + spark-parent 2.0.0-SNAPSHOT ../pom.xml From 34d52d3b66cc9f50e0d5620cba37580716848bed Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:11:46 -0700 Subject: [PATCH 2/8] WIP towards fixing artifact names and build output directories. --- common/network-common/pom.xml | 8 +------- common/network-shuffle/pom.xml | 15 +++------------ common/network-yarn/pom.xml | 11 ++--------- common/sketch/pom.xml | 2 -- common/tags/pom.xml | 5 ----- common/unsafe/pom.xml | 12 +++--------- core/pom.xml | 10 ++++------ examples/pom.xml | 2 -- external/flume-assembly/pom.xml | 2 -- external/flume-sink/pom.xml | 2 -- external/flume/pom.xml | 4 ---- external/kafka-assembly/pom.xml | 2 -- external/kafka/pom.xml | 4 ---- external/kinesis-asl-assembly/pom.xml | 2 -- external/kinesis-asl/pom.xml | 4 ---- graphx/pom.xml | 4 ---- launcher/pom.xml | 12 +----------- mllib-local/pom.xml | 4 ---- mllib/pom.xml | 4 ---- pom.xml | 7 +++++++ project/SparkBuild.scala | 9 +++++++++ project/project/SparkPluginBuild.scala | 6 +++++- repl/pom.xml | 2 -- sql/catalyst/pom.xml | 4 +--- sql/core/pom.xml | 2 -- sql/hive-thriftserver/pom.xml | 2 -- sql/hive/pom.xml | 2 -- streaming/pom.xml | 2 -- tools/pom.xml | 2 -- yarn/pom.xml | 8 +------- 30 files changed, 37 insertions(+), 118 deletions(-) diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 0232bbd887b10..956ab02ea636c 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-network-common_2.11 + spark-network-common jar Spark Project Networking http://spark.apache.org/ @@ -64,10 +64,6 @@ log4j test - - org.apache.spark - spark-test-tags_${scala.binary.version} - org.mockito mockito-core @@ -81,8 +77,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index e3315d9191316..63ce40232613e 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-network-shuffle_2.11 + spark-network-shuffle jar Spark Project Shuffle Streaming Service http://spark.apache.org/ @@ -39,7 +39,7 @@ org.apache.spark - spark-network-common_${scala.binary.version} + spark-network-common ${project.version} @@ -73,15 +73,11 @@ org.apache.spark - spark-network-common_${scala.binary.version} + spark-network-common ${project.version} test-jar test - - org.apache.spark - spark-test-tags_${scala.binary.version} - log4j log4j @@ -93,9 +89,4 @@ test - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index d5aa743f970cc..7263f714d953e 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-network-yarn_2.11 + spark-network-yarn jar Spark Project YARN Shuffle Service http://spark.apache.org/ @@ -43,14 +43,9 @@ org.apache.spark - spark-network-shuffle_${scala.binary.version} + spark-network-shuffle ${project.version} - - org.apache.spark - spark-test-tags_${scala.binary.version} - - org.apache.hadoop @@ -64,8 +59,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 231ca65f0e6bd..c8c139e29cdca 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -43,8 +43,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes diff --git a/common/tags/pom.xml b/common/tags/pom.xml index de6852cf35d5e..e7337ee48ff8c 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -42,9 +42,4 @@ compile - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 0c3cbcd6487f2..ee5fc60292e9b 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-unsafe_2.11 + spark-unsafe jar Spark Project Unsafe http://spark.apache.org/ @@ -37,8 +37,8 @@ - com.twitter - chill_${scala.binary.version} + com.esotericsoftware + kryo-shaded @@ -59,10 +59,6 @@ - - org.apache.spark - spark-test-tags_${scala.binary.version} - org.mockito mockito-core @@ -80,8 +76,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes diff --git a/core/pom.xml b/core/pom.xml index 7f825e0033ab3..e3d98bd8214b3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -61,22 +61,22 @@ org.apache.spark - spark-launcher_${scala.binary.version} + spark-launcher ${project.version} org.apache.spark - spark-network-common_${scala.binary.version} + spark-network-common ${project.version} org.apache.spark - spark-network-shuffle_${scala.binary.version} + spark-network-shuffle ${project.version} org.apache.spark - spark-unsafe_${scala.binary.version} + spark-unsafe ${project.version} @@ -322,8 +322,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/examples/pom.xml b/examples/pom.xml index 4395a6699bcb7..790eb9d659748 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -280,8 +280,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 3fbc5483c7af3..2911019d5a5ef 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -107,8 +107,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index a5fa67a36ed38..87cf48b460412 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -96,8 +96,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.avro diff --git a/external/flume/pom.xml b/external/flume/pom.xml index ac746aafd32ab..a1741439d7423 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -71,8 +71,4 @@ spark-test-tags_${scala.binary.version} - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 378905e057e6d..634458834f37d 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -134,8 +134,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 971bd3b525535..08f30b783485b 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -91,8 +91,4 @@ spark-test-tags_${scala.binary.version} - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 4a3bde2d82e2f..8ec6071c07a8f 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -129,8 +129,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes org.apache.maven.plugins diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index c846dc1348141..94b572c096de8 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -80,8 +80,4 @@ spark-test-tags_${scala.binary.version} - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/graphx/pom.xml b/graphx/pom.xml index 47481d65d7d14..dfcf3e1615ffa 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -75,8 +75,4 @@ spark-test-tags_${scala.binary.version} - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/launcher/pom.xml b/launcher/pom.xml index d428aaaacda65..f46bc186cacfa 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-launcher_2.11 + spark-launcher jar Spark Project Launcher http://spark.apache.org/ @@ -63,11 +63,6 @@ test - - org.apache.spark - spark-test-tags_${scala.binary.version} - - org.apache.hadoop @@ -75,9 +70,4 @@ test - - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 62cc1a49ce322..cc65b7834781d 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -80,8 +80,4 @@ - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/mllib/pom.xml b/mllib/pom.xml index 25f5747aed368..580177704f90e 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -145,8 +145,4 @@ - - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes - diff --git a/pom.xml b/pom.xml index ad27d94806996..94f1597516195 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,8 @@ 8.1.14.v20131031 3.0.0.v201112011016 0.8.0 + + 3.0.3 2.4.0 2.0.8 3.1.2 @@ -272,6 +274,11 @@ ${project.version} test + + com.esotericsoftware + kryo-shaded + ${kryo.version} + com.twitter chill_${scala.binary.version} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index a58dd7e7f125c..540222690e8af 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -172,6 +172,15 @@ object SparkBuild extends PomBuild { publishLocal in MavenCompile <<= publishTask(publishLocalConfiguration in MavenCompile, deliverLocal), publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn, + // Don't append Scala versions to the generated artifacts. We handle this ourselves by doing + // this in the POM. In order for our POM names to work properly, we need to disable SBT's + // normalization so that spark-foo_2.xx isn't escaped to spark-foo_2-xx. + crossPaths := false, + normalizedName := name.value, + // Don't automatically include Scala library dependency. We manage our dependencies via Maven + // POMs and thus do not want SBT to implicitly add additional dependencies. + autoScalaLibrary := false, + javacOptions in (Compile, doc) ++= { val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3) var major = versionParts(0).toInt diff --git a/project/project/SparkPluginBuild.scala b/project/project/SparkPluginBuild.scala index cbb88dc7dd1dd..3ac438a7c6f4c 100644 --- a/project/project/SparkPluginBuild.scala +++ b/project/project/SparkPluginBuild.scala @@ -24,5 +24,9 @@ import sbt.Keys._ */ object SparkPluginDef extends Build { lazy val root = Project("plugins", file(".")) dependsOn(sbtPomReader) - lazy val sbtPomReader = uri("https://github.com/ScrapCodes/sbt-pom-reader.git#ignore_artifact_id") + // This corresponds to https://github.com/ScrapCodes/sbt-pom-reader/commits/test-scoped-depstest-scoped-deps") + // This branch contains https://github.com/sbt/sbt-pom-reader/pull/14, a patch to fix test-jar dependencies + // which has not yet been merged upstream. Once that's merged, we should remove our custom fork and use + // the official sbt-pom-reader release instead; see SPARK-14401 + lazy val sbtPomReader = uri("https://github.com/ScrapCodes/sbt-pom-reader.git#dee910271ccde761837c19ad1d7d450b1dc63a6d") } diff --git a/repl/pom.xml b/repl/pom.xml index 294f4723778e0..0ceacfa86a9a3 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -119,8 +119,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index fd01adcb626b8..6cd9b7d0d0362 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -59,7 +59,7 @@ org.apache.spark - spark-unsafe_${scala.binary.version} + spark-unsafe ${project.version} @@ -81,8 +81,6 @@ - target/scala-${scala.binary.version}/classes - target/scala-${scala.binary.version}/test-classes ${test_classpath} 1 - ${scala.binary.version} 1 ${test.java.home} @@ -1962,7 +1961,6 @@ --> ${test_classpath} 1 - ${scala.binary.version} 1 ${test.java.home} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 540222690e8af..b36660db81d78 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -717,7 +717,6 @@ object TestSettings { "SPARK_DIST_CLASSPATH" -> (fullClasspath in Test).value.files.map(_.getAbsolutePath).mkString(":").stripSuffix(":"), "SPARK_PREPEND_CLASSES" -> "1", - "SPARK_SCALA_VERSION" -> scalaBinaryVersion, "SPARK_TESTING" -> "1", "JAVA_HOME" -> sys.env.get("JAVA_HOME").getOrElse(sys.props("java.home"))), javaOptions in Test += s"-Djava.io.tmpdir=$testTempDir", diff --git a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala index 6c3556a2ee43e..7442b7bc5e5b4 100644 --- a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala +++ b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala @@ -19,7 +19,6 @@ package org.apache.spark.launcher import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import scala.util.Properties /** * Exposes methods from the launcher library that are used by the YARN backend. @@ -31,11 +30,7 @@ private[spark] object YarnCommandBuilderUtils { } def findJarsDir(sparkHome: String): String = { - val scalaVer = Properties.versionNumberString - .split("\\.") - .take(2) - .mkString(".") - CommandBuilderUtils.findJarsDir(sparkHome, scalaVer, true) + CommandBuilderUtils.findJarsDir(sparkHome, true) } /** From f4de68930339227bf414d4a27cbff9222fee322e Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:37:33 -0700 Subject: [PATCH 4/8] Remove more references to target/scala- --- common/network-yarn/pom.xml | 2 +- dev/change-scala-version.sh | 4 ---- dev/make-distribution.sh | 8 ++++---- docs/_plugins/copy_api_dirs.rb | 2 +- docs/building-spark.md | 4 ++-- docs/job-scheduling.md | 2 +- docs/streaming-kinesis-integration.md | 2 +- external/flume-sink/pom.xml | 2 +- .../python/examples/streaming/kinesis_wordcount_asl.py | 2 +- pom.xml | 2 +- python/pyspark/streaming/tests.py | 3 +-- python/run-tests.py | 8 +++----- 12 files changed, 17 insertions(+), 24 deletions(-) diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 7263f714d953e..8084c5d108aee 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -35,7 +35,7 @@ network-yarn provided - ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar + ${project.build.directory}/spark-${project.version}-yarn-shuffle.jar org/spark_project/ diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh index ce6d32300cba4..42213875740f3 100755 --- a/dev/change-scala-version.sh +++ b/dev/change-scala-version.sh @@ -59,7 +59,3 @@ export -f sed_i BASEDIR=$(dirname $0)/.. find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' {}" \; - -# Update source of scaladocs -echo "$BASEDIR/docs/_plugins/copy_api_dirs.rb" -sed_i 's/scala\-'$FROM_VERSION'/scala\-'$TO_VERSION'/' "$BASEDIR/docs/_plugins/copy_api_dirs.rb" diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh index 4f7544f6ea78b..4bb3ef191392b 100755 --- a/dev/make-distribution.sh +++ b/dev/make-distribution.sh @@ -165,17 +165,17 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI echo "Build flags: $@" >> "$DISTDIR/RELEASE" # Copy jars -cp "$SPARK_HOME"/assembly/target/scala*/jars/* "$DISTDIR/jars/" +cp "$SPARK_HOME"/assembly/target/jars/* "$DISTDIR/jars/" # Only create the yarn directory if the yarn artifacts were build. -if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar ]; then +if [ -f "$SPARK_HOME"/common/network-yarn/target/spark-*-yarn-shuffle.jar ]; then mkdir "$DISTDIR"/yarn - cp "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/yarn" + cp "$SPARK_HOME"/common/network-yarn/target/spark-*-yarn-shuffle.jar "$DISTDIR/yarn" fi # Copy examples and dependencies mkdir -p "$DISTDIR/examples/jars" -cp "$SPARK_HOME"/examples/target/scala*/jars/* "$DISTDIR/examples/jars" +cp "$SPARK_HOME"/examples/target/jars/* "$DISTDIR/examples/jars" # Deduplicate jars that have already been packaged as part of the main Spark dependencies. for f in "$DISTDIR/examples/jars/"*; do diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index f926d67e6beaf..1ed41e5f86d6b 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -37,7 +37,7 @@ # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.11/unidoc" + source = "../target/unidoc" dest = "api/scala" puts "Making directory " + dest diff --git a/docs/building-spark.md b/docs/building-spark.md index 40661604af942..a23198d7f7f91 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -52,10 +52,10 @@ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" If you don't run this, you may see errors like the following: - [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_BINARY_VERSION}}/classes... + [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/classes... [ERROR] PermGen space -> [Help 1] - [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_BINARY_VERSION}}/classes... + [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/classes... [ERROR] Java heap space -> [Help 1] You can fix this by setting the `MAVEN_OPTS` variable as discussed before. diff --git a/docs/job-scheduling.md b/docs/job-scheduling.md index 083c020caa5db..814f71088edf5 100644 --- a/docs/job-scheduling.md +++ b/docs/job-scheduling.md @@ -88,7 +88,7 @@ In YARN mode, start the shuffle service on each `NodeManager` as follows: 1. Build Spark with the [YARN profile](building-spark.html). Skip this step if you are using a pre-packaged distribution. 2. Locate the `spark--yarn-shuffle.jar`. This should be under -`$SPARK_HOME/common/network-yarn/target/scala-` if you are building Spark yourself, and under +`$SPARK_HOME/common/network-yarn/target/` if you are building Spark yourself, and under `lib` if you are using a distribution. 2. Add this jar to the classpath of all `NodeManager`s in your cluster. 3. In the `yarn-site.xml` on each node, add `spark_shuffle` to `yarn.nodemanager.aux-services`, diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md index 5b9a7554d2e64..3b6430fd48af1 100644 --- a/docs/streaming-kinesis-integration.md +++ b/docs/streaming-kinesis-integration.md @@ -190,7 +190,7 @@ To run the example,
- bin/spark-submit --jars external/kinesis-asl/target/scala-*/\ + bin/spark-submit --jars external/kinesis-asl/target/\ spark-streaming-kinesis-asl-assembly_*.jar \ external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ [Kinesis app name] [Kinesis stream name] [endpoint URL] [region name] diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 87cf48b460412..a828d4381e797 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -103,7 +103,7 @@ ${avro.version} - ${project.basedir}/target/scala-${scala.binary.version}/src_managed/main/compiled_avro + ${project.basedir}/target/src_managed/main/compiled_avro diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py index 4d7fc9a549bfb..b5675d3790a4d 100644 --- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py +++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py @@ -34,7 +34,7 @@ $ export AWS_SECRET_KEY= # run the example - $ bin/spark-submit -jar external/kinesis-asl/target/scala-*/\ + $ bin/spark-submit -jar external/kinesis-asl/target/\ spark-streaming-kinesis-asl-assembly_*.jar \ external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com diff --git a/pom.xml b/pom.xml index 022021c4d01c0..2df2423d0df39 100644 --- a/pom.xml +++ b/pom.xml @@ -188,7 +188,7 @@ org.spark_project - ${project.build.directory}/scala-${scala.binary.version}/jars + ${project.build.directory}/jars prepare-package diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py index 148bf7e8ff5ce..60fff6c6cc26c 100644 --- a/python/pyspark/streaming/tests.py +++ b/python/pyspark/streaming/tests.py @@ -1469,8 +1469,7 @@ def get_output(_, rdd): def search_jar(dir, name_prefix): # We should ignore the following jars ignored_jar_suffixes = ("javadoc.jar", "sources.jar", "test-sources.jar", "tests.jar") - jars = (glob.glob(os.path.join(dir, "target/scala-*/" + name_prefix + "-*.jar")) + # sbt build - glob.glob(os.path.join(dir, "target/" + name_prefix + "_*.jar"))) # maven build + jars = glob.glob(os.path.join(dir, "target/" + name_prefix + "_*.jar")) return [jar for jar in jars if not jar.endswith(ignored_jar_suffixes)] diff --git a/python/run-tests.py b/python/run-tests.py index 38b3bb84c10be..fb08a15fe8704 100755 --- a/python/run-tests.py +++ b/python/run-tests.py @@ -54,11 +54,9 @@ def print_red(text): LOGGER = logging.getLogger() # Find out where the assembly jars are located. -for scala in ["2.11", "2.10"]: - build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala) - if os.path.isdir(build_dir): - SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*") - break +build_dir = os.path.join(SPARK_HOME, "assembly", "target") +if os.path.isdir(build_dir): + SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*") else: raise Exception("Cannot find assembly build directory, please build Spark first.") From 04161558a1097b634d459117dac66bd661b7cb79 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:42:56 -0700 Subject: [PATCH 5/8] More updates in examples/ --- examples/src/main/python/streaming/direct_kafka_wordcount.py | 2 +- examples/src/main/python/streaming/flume_wordcount.py | 2 +- examples/src/main/python/streaming/kafka_wordcount.py | 2 +- .../org/apache/spark/examples/mllib/BinaryClassification.scala | 2 +- .../scala/org/apache/spark/examples/mllib/Correlations.scala | 2 +- .../org/apache/spark/examples/mllib/LinearRegression.scala | 2 +- .../scala/org/apache/spark/examples/mllib/MovieLensALS.scala | 2 +- .../apache/spark/examples/mllib/MultivariateSummarizer.scala | 2 +- .../scala/org/apache/spark/examples/mllib/SampledRDDs.scala | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/src/main/python/streaming/direct_kafka_wordcount.py b/examples/src/main/python/streaming/direct_kafka_wordcount.py index 7097f7f4502bd..f0260b6539b39 100644 --- a/examples/src/main/python/streaming/direct_kafka_wordcount.py +++ b/examples/src/main/python/streaming/direct_kafka_wordcount.py @@ -24,7 +24,7 @@ and then run the example `$ bin/spark-submit --jars \ - external/kafka-assembly/target/scala-*/spark-streaming-kafka-assembly-*.jar \ + external/kafka-assembly/target/spark-streaming-kafka-assembly-*.jar \ examples/src/main/python/streaming/direct_kafka_wordcount.py \ localhost:9092 test` """ diff --git a/examples/src/main/python/streaming/flume_wordcount.py b/examples/src/main/python/streaming/flume_wordcount.py index d75bc6daac138..a49f8909380da 100644 --- a/examples/src/main/python/streaming/flume_wordcount.py +++ b/examples/src/main/python/streaming/flume_wordcount.py @@ -24,7 +24,7 @@ and then run the example `$ bin/spark-submit --jars \ - external/flume-assembly/target/scala-*/spark-streaming-flume-assembly-*.jar \ + external/flume-assembly/target/spark-streaming-flume-assembly-*.jar \ examples/src/main/python/streaming/flume_wordcount.py \ localhost 12345 """ diff --git a/examples/src/main/python/streaming/kafka_wordcount.py b/examples/src/main/python/streaming/kafka_wordcount.py index 8d697f620f467..2eb204643a3c1 100644 --- a/examples/src/main/python/streaming/kafka_wordcount.py +++ b/examples/src/main/python/streaming/kafka_wordcount.py @@ -24,7 +24,7 @@ and then run the example `$ bin/spark-submit --jars \ - external/kafka-assembly/target/scala-*/spark-streaming-kafka-assembly-*.jar \ + external/kafka-assembly/target/spark-streaming-kafka-assembly-*.jar \ examples/src/main/python/streaming/kafka_wordcount.py \ localhost:2181 test` """ diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala index 2282bd2b7d680..5382317361f50 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala @@ -89,7 +89,7 @@ object BinaryClassification { |For example, the following command runs this app on a synthetic dataset: | | bin/spark-submit --class org.apache.spark.examples.mllib.BinaryClassification \ - | examples/target/scala-*/spark-examples-*.jar \ + | examples/target/spark-examples-*.jar \ | --algorithm LR --regType L2 --regParam 1.0 \ | data/mllib/sample_binary_classification_data.txt """.stripMargin) diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala index e003f35ed399f..c170296345bfb 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala @@ -51,7 +51,7 @@ object Correlations { |For example, the following command runs this app on a synthetic dataset: | | bin/spark-submit --class org.apache.spark.examples.mllib.Correlations \ - | examples/target/scala-*/spark-examples-*.jar \ + | examples/target/spark-examples-*.jar \ | --input data/mllib/sample_linear_regression_data.txt """.stripMargin) } diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala index f87611f5d4613..71ef255d98312 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala @@ -76,7 +76,7 @@ object LinearRegression { |For example, the following command runs this app on a synthetic dataset: | | bin/spark-submit --class org.apache.spark.examples.mllib.LinearRegression \ - | examples/target/scala-*/spark-examples-*.jar \ + | examples/target/spark-examples-*.jar \ | data/mllib/sample_linear_regression_data.txt """.stripMargin) } diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala index 09750e53cb169..2ce3dec6cbb94 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala @@ -83,7 +83,7 @@ object MovieLensALS { |For example, the following command runs this app on a synthetic dataset: | | bin/spark-submit --class org.apache.spark.examples.mllib.MovieLensALS \ - | examples/target/scala-*/spark-examples-*.jar \ + | examples/target/spark-examples-*.jar \ | --rank 5 --numIterations 20 --lambda 1.0 --kryo \ | data/mllib/sample_movielens_data.txt """.stripMargin) diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala index 3c598172dadf0..950ddb00d67e4 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala @@ -52,7 +52,7 @@ object MultivariateSummarizer { |For example, the following command runs this app on a synthetic dataset: | | bin/spark-submit --class org.apache.spark.examples.mllib.MultivariateSummarizer \ - | examples/target/scala-*/spark-examples-*.jar \ + | examples/target/spark-examples-*.jar \ | --input data/mllib/sample_linear_regression_data.txt """.stripMargin) } diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala index 0da4005977d1a..f9ecc9e4dac97 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala @@ -48,7 +48,7 @@ object SampledRDDs { |For example, the following command runs this app: | | bin/spark-submit --class org.apache.spark.examples.mllib.SampledRDDs \ - | examples/target/scala-*/spark-examples-*.jar + | examples/target/spark-examples-*.jar """.stripMargin) } From 449e93464b16b97c8af3806b73d96059cba5b45c Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:49:11 -0700 Subject: [PATCH 6/8] Remove Scala download from build/mvn --- build/mvn | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/build/mvn b/build/mvn index eb42552fc499e..34eee24805405 100755 --- a/build/mvn +++ b/build/mvn @@ -93,25 +93,6 @@ install_zinc() { ZINC_BIN="${_DIR}/${zinc_path}" } -# Determine the Scala version from the root pom.xml file, set the Scala URL, -# and, with that, download the specific version of Scala necessary under -# the build/ folder -install_scala() { - # determine the Scala version used in Spark - local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | \ - head -1 | cut -f2 -d'>' | cut -f1 -d'<'` - local scala_bin="${_DIR}/scala-${scala_version}/bin/scala" - local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com} - - install_app \ - "${TYPESAFE_MIRROR}/scala/${scala_version}" \ - "scala-${scala_version}.tgz" \ - "scala-${scala_version}/bin/scala" - - SCALA_COMPILER="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-compiler.jar" - SCALA_LIBRARY="$(cd "$(dirname "${scala_bin}")/../lib" && pwd)/scala-library.jar" -} - # Setup healthy defaults for the Zinc port if none were provided from # the environment ZINC_PORT=${ZINC_PORT:-"3030"} @@ -132,7 +113,6 @@ fi # Install the proper version of Scala and Zinc for the build install_zinc -install_scala # Reset the current working directory cd "${_CALLING_DIR}" @@ -142,9 +122,7 @@ cd "${_CALLING_DIR}" if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"} "${ZINC_BIN}" -shutdown -port ${ZINC_PORT} - "${ZINC_BIN}" -start -port ${ZINC_PORT} \ - -scala-compiler "${SCALA_COMPILER}" \ - -scala-library "${SCALA_LIBRARY}" &>/dev/null + "${ZINC_BIN}" -start -port ${ZINC_PORT} &>/dev/null fi # Set any `mvn` options if not already present From acd738f33c1210bfae44d631c86fbd830a95d7d1 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:53:11 -0700 Subject: [PATCH 7/8] Update spark-sketch artifactid and pom. --- common/sketch/pom.xml | 9 +-------- sql/core/pom.xml | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index c8c139e29cdca..c342b1b576383 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -27,7 +27,7 @@ org.apache.spark - spark-sketch_2.11 + spark-sketch jar Spark Project Sketch http://spark.apache.org/ @@ -35,13 +35,6 @@ sketch - - - org.apache.spark - spark-test-tags_${scala.binary.version} - - - diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 8c537077e7a2a..35f4389cbd99d 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -44,7 +44,7 @@ org.apache.spark - spark-sketch_2.11 + spark-sketch ${project.version} From 944e8ab3da77a406fec9a8c1da118299f228a3e0 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 Apr 2016 13:53:53 -0700 Subject: [PATCH 8/8] More cleanup in make-distribution.sh --- dev/make-distribution.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh index 4bb3ef191392b..2102e37614921 100755 --- a/dev/make-distribution.sh +++ b/dev/make-distribution.sh @@ -117,9 +117,6 @@ if [ ! "$(command -v "$MVN")" ] ; then fi VERSION=$("$MVN" help:evaluate -Dexpression=project.version $@ 2>/dev/null | grep -v "INFO" | tail -n 1) -SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version $@ 2>/dev/null\ - | grep -v "INFO"\ - | tail -n 1) SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\ | grep -v "INFO"\ | tail -n 1)