From 4e96c0153063b35fc03e497f28292a97832e81d4 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:03:30 -0700 Subject: [PATCH 01/25] Add YARN/Stable compiled classes to the CLASSPATH. The change adds the `./yarn/stable/target//classes` to the _Classpath_ when a _dependencies_ assembly is available at the assembly directory. Why is this change necessary? Ease the development features and bug-fixes for Spark-YARN. [ticket: X] : NA Author : bernardo.gomezpalacio@gmail.com Reviewer : ? Testing : ? --- bin/compute-classpath.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 2a2bb376fd71f..2efec362b1df5 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -44,6 +44,7 @@ if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes" + CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes" DEPS_ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar` CLASSPATH="$CLASSPATH:$DEPS_ASSEMBLY_JAR" From 1342886a396be00eda9449c6d84155dfecf954c8 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:46:44 -0700 Subject: [PATCH 02/25] The `spark-class` shell now ignores non jar files in the assembly directory. Why is this change necessary? While developing in Spark I found myself rebuilding either the dependencies assembly or the full spark assembly. I kept running into the case of having both the dep-assembly and full-assembly in the same directory and getting an error when I called either `spark-shell` or `spark-submit`. Quick fix: move either of them as a .bkp file depending on the development work flow you are executing at the moment and enabling the `spark-class` to ignore non-jar files. An other option could be to move the "offending" jar to a different directory but in my opinion keeping them in there is a bit tidier. e.g. ``` ll ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-deps.jar spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0.jar.bkp ``` [ticket: X] : ? --- bin/spark-class | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/spark-class b/bin/spark-class index 1b0d309cc5b1c..ff03f1fda6e9e 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -109,8 +109,8 @@ export JAVA_OPTS if [ ! -f "$FWDIR/RELEASE" ]; then # Exit if the user hasn't compiled Spark - num_jars=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) - jars_list=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep "spark-assembly.*hadoop.*.jar") + num_jars=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep -E "spark-assembly.*hadoop.*.jar$" | wc -l) + jars_list=$(ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/ | grep -E "spark-assembly.*hadoop.*.jar$") if [ "$num_jars" -eq "0" ]; then echo "Failed to find Spark assembly in $FWDIR/assembly/target/scala-$SCALA_VERSION/" >&2 echo "You need to build Spark with 'sbt/sbt assembly' before running this program." >&2 From ddf2547aa2aea8155f8d6c0386e2cb37bcf61537 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 14:53:23 -0700 Subject: [PATCH 03/25] The `spark-shell` option `--log-conf` also enables the SPARK_PRINT_LAUNCH_COMMAND . Why is this change necessary? Most likely when enabling the `--log-conf` through the `spark-shell` you are also interested on the full invocation of the java command including the _classpath_ and extended options. e.g. ``` INFO: Base Directory set to /Users/bernardo/work/github/berngp/spark INFO: Spark Master is yarn-client INFO: Spark REPL options -Dspark.logConf=true Spark Command: /Library/Java/JavaVirtualMachines/jdk1.8.0.jdk/Contents/Home/bin/java -cp :/Users/bernardo/work/github/berngp/spark/conf:/Users/bernardo/work/github/berngp/spark/core/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/repl/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/mllib/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/bagel/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/graphx/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/streaming/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/tools/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/catalyst/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/core/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/sql/hive/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/yarn/stable/target/scala-2.10/classes:/Users/bernardo/work/github/berngp/spark/assembly/target/scala-2.10/spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-deps.jar:/usr/local/Cellar/hadoop/2.2.0/libexec/etc/hadoop -XX:ErrorFile=/tmp/spark-shell-hs_err_pid.log -XX:HeapDumpPath=/tmp/spark-shell-java_pid.hprof -XX:-HeapDumpOnOutOfMemoryError -XX:-PrintGC -XX:-PrintGCDetails -XX:-PrintGCTimeStamps -XX:-PrintTenuringDistribution -XX:-PrintAdaptiveSizePolicy -XX:GCLogFileSize=1024K -XX:-UseGCLogFileRotation -Xloggc:/tmp/spark-shell-gc.log -XX:+UseConcMarkSweepGC -Dspark.cleaner.ttl=10000 -Dspark.driver.host=33.33.33.1 -Dspark.logConf=true -Djava.library.path= -Xms400M -Xmx400M org.apache.spark.repl.Main ``` [ticket: X] : ? --- bin/spark-shell | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/spark-shell b/bin/spark-shell index ea12d256b23a1..cc9d1ea201469 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -123,6 +123,7 @@ function set_dm(){ function set_spark_log_conf(){ SPARK_REPL_OPTS="$SPARK_REPL_OPTS -Dspark.logConf=$1" + export SPARK_PRINT_LAUNCH_COMMAND=1 } function set_spark_master(){ From 22045394955992c2c8dfe0e1040c6bb972be6ce4 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 15:15:23 -0700 Subject: [PATCH 04/25] Root is now Spark and qualify the assembly if it was built with YARN. Why is this change necessary? Renamed the SBT "root" project to "spark" to enhance readability. Currently the assembly is qualified with the Hadoop Version but not if YARN has been enabled or not. This change qualifies the assembly such that it is easy to identify if YARN was enabled. e.g ``` ./make-distribution.sh --hadoop 2.3.0 --with-yarn ls -l ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0-yarn.jar ``` vs ``` ./make-distribution.sh --hadoop 2.3.0 ls -l ./assembly/target/scala-2.10 spark-assembly-1.0.0-SNAPSHOT-hadoop2.3.0.jar ``` [ticket: X] : ? --- project/SparkBuild.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 09b527c76a5ae..f6e8a6ade161e 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -52,7 +52,7 @@ object SparkBuild extends Build { val SCALAC_JVM_VERSION = "jvm-1.6" val JAVAC_JVM_VERSION = "1.6" - lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*) + lazy val root = Project("spark", file("."), settings = rootSettings) aggregate(allProjects: _*) lazy val core = Project("core", file("core"), settings = coreSettings) @@ -409,7 +409,7 @@ object SparkBuild extends Build { def catalystSettings = sharedSettings ++ Seq( name := "catalyst", // The mechanics of rewriting expression ids to compare trees in some test cases makes - // assumptions about the the expression ids being contiguious. Running tests in parallel breaks + // assumptions about the the expression ids being contiguous. Running tests in parallel breaks // this non-deterministically. TODO: FIX THIS. parallelExecution in Test := false, libraryDependencies ++= Seq( @@ -516,7 +516,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, + jarName in assembly <<= version map { v => s"spark-assembly-${v}-hadoop${hadoopVersion}${if (isYarnEnabled) "-yarn" else ""}.jar" }, jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From 889bf4ed742ed3d06cb62276ef554f2f37b53ee6 Mon Sep 17 00:00:00 2001 From: Bernardo Gomez Palacio Date: Tue, 15 Apr 2014 17:08:27 -0700 Subject: [PATCH 05/25] Upgrade the Maven Build to YARN 2.3.0. Upgraded to YARN 2.3.0, removed unnecessary `relativePath` values and removed incorrect version for the "org.apache.hadoop:hadoop-client" dependency at yarn/pom.xml. --- pom.xml | 3 ++- yarn/alpha/pom.xml | 1 - yarn/pom.xml | 4 +--- yarn/stable/pom.xml | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index cd204376de5db..0fd3d227a066a 100644 --- a/pom.xml +++ b/pom.xml @@ -836,7 +836,8 @@ yarn 2 - 2.2.0 + 2.3.0 + 2.3.0 2.5.0 diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index d0aeaceb0d23c..1892d2111713c 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -21,7 +21,6 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT - ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index 3342cb65edcd1..f860958963841 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -21,14 +21,13 @@ org.apache.spark spark-parent 1.0.0-SNAPSHOT - ../pom.xml org.apache.spark yarn-parent_2.10 pom Spark Project YARN Parent POM - + org.apache.spark @@ -50,7 +49,6 @@ org.apache.hadoop hadoop-client - ${yarn.version} org.scalatest diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index e7915d12aef63..22ba114612132 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -21,7 +21,6 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT - ../pom.xml From f1c7535fe6e97e1d5ebf8adcac01d82c794a01f8 Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 01:48:01 +0800 Subject: [PATCH 06/25] =?UTF-8?q?Improved=20build=20configuration=20?= =?UTF-8?q?=E2=85=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/building-with-maven.md | 9 ++- pom.xml | 133 ++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 70 deletions(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index a5e5303467401..5eb85f05a4d45 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -45,17 +45,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile: # Apache Hadoop 2.0.5-alpha - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v2 - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -Dyarn.version=2.0.0-cdh4.2.0 -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package # Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer - $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package + $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package + # Different versions of HDFS vs YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version= 0.23.7 -DskipTests clean package + ## Spark Tests in Maven ## Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`. diff --git a/pom.xml b/pom.xml index 560bd364ff918..73de1cc385c5b 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 org.apache @@ -119,7 +120,7 @@ 1.2.17 1.0.4 2.4.1 - 0.23.7 + ${hadoop.version} 0.94.6 0.12.0 1.3.2 @@ -135,7 +136,8 @@ - maven-repo + maven-repo + Maven Repository http://repo.maven.apache.org/maven2 @@ -558,64 +560,7 @@ jets3t 0.7.1 - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - org.codehaus.jackson @@ -850,12 +795,6 @@ yarn - - - org.apache.avro - avro - - @@ -896,13 +835,73 @@ 2 2.3.0 - 2.3.0 2.5.0 yarn + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + From 8540e83420a7d11394501f88872860d32ebc3f1b Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 09:37:06 +0800 Subject: [PATCH 07/25] review commit --- docs/building-with-maven.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 5eb85f05a4d45..e447dfea3bac9 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -56,8 +56,8 @@ For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package - # Different versions of HDFS vs YARN. - $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version= 0.23.7 -DskipTests clean package + # Different versions of HDFS and YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package ## Spark Tests in Maven ## From c4c6e45ff134663196e07fbd3a812af57b8ea95c Mon Sep 17 00:00:00 2001 From: witgo Date: Wed, 30 Apr 2014 23:23:08 +0800 Subject: [PATCH 08/25] review commit --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 45b4a753ef077..f609343dd1baf 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -569,7 +569,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => s"spark-assembly-${v}-hadoop${hadoopVersion}${if (isYarnEnabled) "-yarn" else ""}.jar" }, + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From e1a7e0020f3c42b9e7dae07cabd072f0dd38d463 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 14:34:57 +0800 Subject: [PATCH 09/25] improve travis tests coverage --- .gitignore | 1 + .jvmopts | 5 +++++ .travis.yml | 28 +++++++++++++++++++++------- core/pom.xml | 29 ----------------------------- pom.xml | 5 +++++ repl/pom.xml | 29 ----------------------------- yarn/pom.xml | 29 ----------------------------- 7 files changed, 32 insertions(+), 94 deletions(-) create mode 100644 .jvmopts diff --git a/.gitignore b/.gitignore index 857e9feb953bd..a509bfbb495e3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ conf/spark-env.sh conf/streaming-env.sh conf/log4j.properties conf/spark-defaults.conf +conf/*.xml docs/_site docs/api target/ diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 0000000000000..86db0ab9b6ddc --- /dev/null +++ b/.jvmopts @@ -0,0 +1,5 @@ +-Xmx3g +-Xss2M +-XX:+CMSClassUnloadingEnabled +-XX:MaxPermSize=512M +-XX:ReservedCodeCacheSize=512m \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8ebd0d68429fc..4895faed78d28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,18 +15,32 @@ language: scala scala: - - "2.10.3" + - "2.10.4" jdk: + - oraclejdk8 - oraclejdk7 - env: - matrix: - - TEST="scalastyle assembly/assembly" - - TEST="catalyst/test sql/test streaming/test mllib/test graphx/test bagel/test" - - TEST=hive/test + - openjdk6 cache: directories: - $HOME/.m2 - $HOME/.ivy2 - $HOME/.sbt + env: + matrix: + - SPARK_HADOOP_VERSION=1.2.1 + - SPARK_HADOOP_VERSION=2.4.0 + before_script: + - sudo apt-get install libgfortran3 script: - - "sbt ++$TRAVIS_SCALA_VERSION $TEST" + # TODO: Cannot get hive/test to pass + > + if [[ $SPARK_HADOOP_VERSION = '2.4.0' ]]; then + mvn clean package -DskipTests -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION + num_jars=$(ls "$FWDIR"/assembly/target/scala-2.10/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) + if [ "$num_jars" -eq "0" ]; then + exit -1 + fi + mvn test -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION -am -pl mllib -pl bagel -pl sql/catalyst -pl yarn + else + SPARK_HADOOP_VERSION=$SPARK_HADOOP_VERSION sbt ++$TRAVIS_SCALA_VERSION assembly/assembly sql/test graphx/test streaming/test + fi \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index 822b5b1dd7cc2..1f71bb95f8571 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -254,35 +254,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin diff --git a/pom.xml b/pom.xml index 73de1cc385c5b..e1e0bce3929d5 100644 --- a/pom.xml +++ b/pom.xml @@ -682,6 +682,11 @@ ${project.build.directory}/SparkTestSuite.txt -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m + + ${basedir}/.. + 1 + ${spark.classpath} + diff --git a/repl/pom.xml b/repl/pom.xml index b761a176ce256..8539a9394ddd9 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -92,35 +92,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin diff --git a/yarn/pom.xml b/yarn/pom.xml index a10436c07d919..35b5090a102a4 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -112,35 +112,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - - - test - - run - - - true - - - - - - - - - - - - - - - - - - org.scalatest scalatest-maven-plugin From effe79cd8c018c31be31c53260fad59fa50a77a9 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 14:46:04 +0800 Subject: [PATCH 10/25] missing "," --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f609343dd1baf..03e25acf6f1b8 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -569,7 +569,7 @@ object SparkBuild extends Build { libraryDependencies += "net.sf.py4j" % "py4j" % "0.8.1", name := "spark-assembly", assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings From 9ea1af926c3ccf36c59e13c103ec6052c032f77e Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 15:04:53 +0800 Subject: [PATCH 11/25] add the dependency of commons-lang --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index e1e0bce3929d5..76e0e92a4bc12 100644 --- a/pom.xml +++ b/pom.xml @@ -373,6 +373,11 @@ commons-net 2.2 + + commons-lang + commons-lang + 2.5 + io.netty netty-all From 0ed124dc0e453a0a59d3c387651be970859a9a0a Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 16:07:45 +0800 Subject: [PATCH 12/25] SPARK-1693: Most of the tests throw a java.lang.SecurityException when spark built for hadoop 2.3.0 , 2.4.0 --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 76e0e92a4bc12..1a7f590cc333e 100644 --- a/pom.xml +++ b/pom.xml @@ -888,6 +888,10 @@ org.jboss.netty netty + + javax.servlet + servlet-api + @@ -908,6 +912,10 @@ org.jboss.netty netty + + javax.servlet + servlet-api + From 03b136fe1ad64ee12ffe90bacfe715a2ad87b2c2 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 16:27:11 +0800 Subject: [PATCH 13/25] revert .travis.yml --- .jvmopts | 5 ----- .travis.yml | 28 +++++++--------------------- 2 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 .jvmopts diff --git a/.jvmopts b/.jvmopts deleted file mode 100644 index 86db0ab9b6ddc..0000000000000 --- a/.jvmopts +++ /dev/null @@ -1,5 +0,0 @@ --Xmx3g --Xss2M --XX:+CMSClassUnloadingEnabled --XX:MaxPermSize=512M --XX:ReservedCodeCacheSize=512m \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 4895faed78d28..a37e958c68095 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,32 +15,18 @@ language: scala scala: - - "2.10.4" + - "2.10.3" jdk: - - oraclejdk8 - oraclejdk7 - - openjdk6 + env: + matrix: + - TEST="scalastyle assembly/assembly" + - TEST="catalyst/test sql/test streaming/test mllib/test graphx/test bagel/test" + - TEST=hive/test cache: directories: - $HOME/.m2 - $HOME/.ivy2 - $HOME/.sbt - env: - matrix: - - SPARK_HADOOP_VERSION=1.2.1 - - SPARK_HADOOP_VERSION=2.4.0 - before_script: - - sudo apt-get install libgfortran3 script: - # TODO: Cannot get hive/test to pass - > - if [[ $SPARK_HADOOP_VERSION = '2.4.0' ]]; then - mvn clean package -DskipTests -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION - num_jars=$(ls "$FWDIR"/assembly/target/scala-2.10/ | grep "spark-assembly.*hadoop.*.jar" | wc -l) - if [ "$num_jars" -eq "0" ]; then - exit -1 - fi - mvn test -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION -Dyarn.version=$SPARK_HADOOP_VERSION -am -pl mllib -pl bagel -pl sql/catalyst -pl yarn - else - SPARK_HADOOP_VERSION=$SPARK_HADOOP_VERSION sbt ++$TRAVIS_SCALA_VERSION assembly/assembly sql/test graphx/test streaming/test - fi \ No newline at end of file + - "sbt ++$TRAVIS_SCALA_VERSION $TEST" \ No newline at end of file From d3488c6e316657ca1436b8cc148cfb9102747658 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 17:19:15 +0800 Subject: [PATCH 14/25] Add the missing yarn dependencies --- pom.xml | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pom.xml b/pom.xml index 1a7f590cc333e..74567d723f323 100644 --- a/pom.xml +++ b/pom.xml @@ -805,6 +805,76 @@ yarn + + + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + From 779ae5d71992da80a20259978b4cd900110c8f30 Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 22:58:12 +0800 Subject: [PATCH 15/25] Fix SPARK-1693: Dependent on multiple versions of servlet-api jars lead to throw an SecurityException when Spark built for hadoop 2.3.0 , 2.4.0 --- bagel/pom.xml | 4 ++++ core/pom.xml | 4 ++++ examples/pom.xml | 4 ++++ graphx/pom.xml | 4 ++++ mllib/pom.xml | 4 ++++ pom.xml | 13 +++++++++++++ project/SparkBuild.scala | 22 ++++++++++++---------- repl/pom.xml | 4 ++++ streaming/pom.xml | 4 ++++ 9 files changed, 53 insertions(+), 10 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index 355f437c5b16a..adb6916482132 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scalatest scalatest_${scala.binary.version} diff --git a/core/pom.xml b/core/pom.xml index 1f71bb95f8571..54b53f9d4260c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -65,6 +65,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + com.google.guava guava diff --git a/examples/pom.xml b/examples/pom.xml index e1fc149d87f17..f6b3f8150b968 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -120,6 +120,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + com.twitter algebird-core_${scala.binary.version} diff --git a/graphx/pom.xml b/graphx/pom.xml index dc108d2fe7fbd..f8abc7f3c8ac8 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -46,6 +46,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scalatest scalatest_${scala.binary.version} diff --git a/mllib/pom.xml b/mllib/pom.xml index cdd33dbb7970d..947d0d919d64a 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.jblas jblas diff --git a/pom.xml b/pom.xml index 74567d723f323..df40e18089910 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,19 @@ org.eclipse.jetty jetty-server ${jetty.version} + + + org.eclipse.jetty.orbit + javax.servlet + + + + + + javax.servlet + javax.servlet-api + 3.0.1 com.google.guava diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 03e25acf6f1b8..1b52d596c4497 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -266,16 +266,17 @@ object SparkBuild extends Build { */ libraryDependencies ++= Seq( - "io.netty" % "netty-all" % "4.0.17.Final", - "org.eclipse.jetty" % "jetty-server" % jettyVersion, - "org.eclipse.jetty" % "jetty-util" % jettyVersion, - "org.eclipse.jetty" % "jetty-plus" % jettyVersion, - "org.eclipse.jetty" % "jetty-security" % jettyVersion, - "org.scalatest" %% "scalatest" % "1.9.1" % "test", - "org.scalacheck" %% "scalacheck" % "1.10.0" % "test", - "com.novocode" % "junit-interface" % "0.10" % "test", - "org.easymock" % "easymock" % "3.1" % "test", - "org.mockito" % "mockito-all" % "1.8.5" % "test" + "io.netty" % "netty-all" % "4.0.17.Final", + "org.eclipse.jetty" % "jetty-server" % jettyVersion excludeAll(excludeJettyServlet), + "javax.servlet" % "javax.servlet-api" % "3.0.1", + "org.eclipse.jetty" % "jetty-util" % jettyVersion, + "org.eclipse.jetty" % "jetty-plus" % jettyVersion, + "org.eclipse.jetty" % "jetty-security" % jettyVersion, + "org.scalatest" %% "scalatest" % "1.9.1" % "test", + "org.scalacheck" %% "scalacheck" % "1.10.0" % "test", + "com.novocode" % "junit-interface" % "0.10" % "test", + "org.easymock" % "easymock" % "3.1" % "test", + "org.mockito" % "mockito-all" % "1.8.5" % "test" ), testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), @@ -315,6 +316,7 @@ object SparkBuild extends Build { val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") val excludeJruby = ExclusionRule(organization = "org.jruby") val excludeThrift = ExclusionRule(organization = "org.apache.thrift") + val excludeJettyServlet= ExclusionRule(organization = "org.eclipse.jetty.orbit") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { diff --git a/repl/pom.xml b/repl/pom.xml index 8539a9394ddd9..e9d812ee8030f 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -58,6 +58,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scala-lang scala-compiler diff --git a/streaming/pom.xml b/streaming/pom.xml index 6435224a14674..ecbafd833418c 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -41,6 +41,10 @@ org.eclipse.jetty jetty-server + + javax.servlet + javax.servlet-api + org.scala-lang scala-library From 27bd4268a7d0c7300182f43d4edc59ad8154aa0c Mon Sep 17 00:00:00 2001 From: witgo Date: Thu, 1 May 2014 23:01:42 +0800 Subject: [PATCH 16/25] review commit --- yarn/alpha/pom.xml | 1 + yarn/pom.xml | 1 + yarn/stable/pom.xml | 1 + 3 files changed, 3 insertions(+) diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index 14e5bfc518b0a..e076ca1d44b97 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -21,6 +21,7 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark diff --git a/yarn/pom.xml b/yarn/pom.xml index 35b5090a102a4..1a2729466acd6 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -21,6 +21,7 @@ org.apache.spark spark-parent 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index 99a04c2ab9cc8..0780f251b595c 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -21,6 +21,7 @@ org.apache.spark yarn-parent_2.10 1.0.0-SNAPSHOT + ../pom.xml org.apache.spark From 54a86b03b5b0b3f90f2b3c78cb9456a0ec81976b Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 2 May 2014 14:22:36 +0800 Subject: [PATCH 17/25] review commit --- pom.xml | 3 +-- repl/pom.xml | 1 - yarn/pom.xml | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index df40e18089910..e9f51abfc85da 100644 --- a/pom.xml +++ b/pom.xml @@ -701,9 +701,8 @@ -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m - ${basedir}/.. + ${session.executionRootDirectory} 1 - ${spark.classpath} diff --git a/repl/pom.xml b/repl/pom.xml index e9d812ee8030f..c686b2f978ef0 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -102,7 +102,6 @@ ${basedir}/.. - 1 diff --git a/yarn/pom.xml b/yarn/pom.xml index 1a2729466acd6..b8714b54502eb 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -119,7 +119,6 @@ ${basedir}/../.. - 1 ${spark.classpath} From 882e35d354e98b8a416583038b72a49d0a331d8e Mon Sep 17 00:00:00 2001 From: witgo Date: Fri, 2 May 2014 14:24:05 +0800 Subject: [PATCH 18/25] review commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e9f51abfc85da..ad9a2ca70e9eb 100644 --- a/pom.xml +++ b/pom.xml @@ -926,7 +926,7 @@ yarn 2 - 2.3.0 + 2.2.0 2.5.0 From 31451dfe66e14f6ccf7daa32287190c7ef7780ca Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 11:15:32 +0800 Subject: [PATCH 19/25] Compile hive optional --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad9a2ca70e9eb..71def98165f08 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,6 @@ streaming sql/catalyst sql/core - sql/hive repl assembly external/twitter @@ -1049,6 +1048,12 @@ + + hive + + sql/hive + + From 5fb961fdd7b601c937074960641ef17fbd08a7b5 Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 15:54:19 +0800 Subject: [PATCH 20/25] revert exclusion org.eclipse.jetty.orbit:javax.servlet --- bagel/pom.xml | 4 ---- core/pom.xml | 4 ---- examples/pom.xml | 4 ---- graphx/pom.xml | 4 ---- mllib/pom.xml | 4 ---- pom.xml | 15 ++------------- repl/pom.xml | 4 ---- streaming/pom.xml | 4 ---- 8 files changed, 2 insertions(+), 41 deletions(-) diff --git a/bagel/pom.xml b/bagel/pom.xml index adb6916482132..355f437c5b16a 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scalatest scalatest_${scala.binary.version} diff --git a/core/pom.xml b/core/pom.xml index 54b53f9d4260c..1f71bb95f8571 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -65,10 +65,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - com.google.guava guava diff --git a/examples/pom.xml b/examples/pom.xml index f6b3f8150b968..e1fc149d87f17 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -120,10 +120,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - com.twitter algebird-core_${scala.binary.version} diff --git a/graphx/pom.xml b/graphx/pom.xml index f8abc7f3c8ac8..dc108d2fe7fbd 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -46,10 +46,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scalatest scalatest_${scala.binary.version} diff --git a/mllib/pom.xml b/mllib/pom.xml index 947d0d919d64a..cdd33dbb7970d 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.jblas jblas diff --git a/pom.xml b/pom.xml index 71def98165f08..e79972cbb42c3 100644 --- a/pom.xml +++ b/pom.xml @@ -214,19 +214,6 @@ org.eclipse.jetty jetty-server ${jetty.version} - - - org.eclipse.jetty.orbit - javax.servlet - - - - - - javax.servlet - javax.servlet-api - 3.0.1 com.google.guava @@ -854,6 +841,8 @@ org.jboss.netty netty + javax.servlet servlet-api diff --git a/repl/pom.xml b/repl/pom.xml index c686b2f978ef0..bcdb24b040cc8 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -58,10 +58,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scala-lang scala-compiler diff --git a/streaming/pom.xml b/streaming/pom.xml index ecbafd833418c..6435224a14674 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -41,10 +41,6 @@ org.eclipse.jetty jetty-server - - javax.servlet - javax.servlet-api - org.scala-lang scala-library From a5ff7d13f523a6494842f61982f706ada4f9a63a Mon Sep 17 00:00:00 2001 From: witgo Date: Sat, 3 May 2014 18:50:53 +0800 Subject: [PATCH 21/25] revert exclusion org.eclipse.jetty.orbit:javax.servlet --- .travis.yml | 2 +- project/SparkBuild.scala | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a37e958c68095..8ebd0d68429fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,4 +29,4 @@ - $HOME/.ivy2 - $HOME/.sbt script: - - "sbt ++$TRAVIS_SCALA_VERSION $TEST" \ No newline at end of file + - "sbt ++$TRAVIS_SCALA_VERSION $TEST" diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1b52d596c4497..b537bc7ed579e 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -267,8 +267,7 @@ object SparkBuild extends Build { libraryDependencies ++= Seq( "io.netty" % "netty-all" % "4.0.17.Final", - "org.eclipse.jetty" % "jetty-server" % jettyVersion excludeAll(excludeJettyServlet), - "javax.servlet" % "javax.servlet-api" % "3.0.1", + "org.eclipse.jetty" % "jetty-server" % jettyVersion, "org.eclipse.jetty" % "jetty-util" % jettyVersion, "org.eclipse.jetty" % "jetty-plus" % jettyVersion, "org.eclipse.jetty" % "jetty-security" % jettyVersion, @@ -316,7 +315,6 @@ object SparkBuild extends Build { val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi") val excludeJruby = ExclusionRule(organization = "org.jruby") val excludeThrift = ExclusionRule(organization = "org.apache.thrift") - val excludeJettyServlet= ExclusionRule(organization = "org.eclipse.jetty.orbit") def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark", version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = { From 427d499247b0e3286270da322b547a61f8fe0a89 Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 10:31:22 +0800 Subject: [PATCH 22/25] merge master --- pom.xml | 224 +++++++++++++++++++++----------------------------------- 1 file changed, 84 insertions(+), 140 deletions(-) diff --git a/pom.xml b/pom.xml index 38b7a60ad57c8..97b8eca44a9b5 100644 --- a/pom.xml +++ b/pom.xml @@ -546,6 +546,90 @@ + + org.apache.hadoop + hadoop-yarn-api + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-common + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + + + org.apache.hadoop + hadoop-yarn-client + ${yarn.version} + + + asm + asm + + + org.ow2.asm + asm + + + org.jboss.netty + netty + + + javax.servlet + servlet-api + + + org.codehaus.jackson @@ -844,76 +928,6 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - @@ -921,76 +935,6 @@ yarn - - - - org.apache.hadoop - hadoop-yarn-api - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - - - org.apache.hadoop - hadoop-yarn-common - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - - org.apache.hadoop - hadoop-yarn-client - ${yarn.version} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - - - From 4cc0c902a923f31da10a5b110a9bb99a0b717c5b Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 13:35:02 +0800 Subject: [PATCH 23/25] revert profile hive --- pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 97b8eca44a9b5..34dc10b527a9b 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ streaming sql/catalyst sql/core + sql/hive repl assembly external/twitter @@ -880,13 +881,6 @@ - - hive - - sql/hive - - - From 4277fedc6ff55c6cca41e4f5d2e6f88f346977af Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 17:01:02 +0800 Subject: [PATCH 24/25] review commit --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index 34dc10b527a9b..e02af0201f79b 100644 --- a/pom.xml +++ b/pom.xml @@ -355,11 +355,6 @@ commons-net 2.2 - - commons-lang - commons-lang - 2.5 - io.netty netty-all From 31c64093abd990ec812cad5afb902d40cc49737b Mon Sep 17 00:00:00 2001 From: witgo Date: Mon, 12 May 2014 17:01:54 +0800 Subject: [PATCH 25/25] review commit --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a471c28d6f83b..ad72588b472d6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ conf/spark-env.sh conf/streaming-env.sh conf/log4j.properties conf/spark-defaults.conf -conf/*.xml docs/_site docs/api target/