From 6184a007fd64f4bba58e6227ca51d0b78dc24010 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 15:32:11 -0700 Subject: [PATCH 01/14] SPARK-9545: Use Maven in prb if title contains "[maven-test]" This is just some small glue code to actually make use of the AMPLAB_JENKINS_BUILD_TOOL switch. As far as I can tell, we actually don't currently use the Maven support in the tool. This patch switches to Maven when the PR title contains "maven-test". There are a few small other pieces of cleanup in the patch as well. --- dev/run-tests-jenkins | 8 ++++++-- dev/run-tests.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index c4d39d95d5890..9742f213a33e4 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -164,8 +164,9 @@ pr_message="" current_pr_head="`git rev-parse HEAD`" echo "HEAD: `git rev-parse HEAD`" -echo "GHPRB: $ghprbActualCommit" -echo "SHA1: $sha1" +echo "\$ghprbActualCommit: $ghprbActualCommit" +echo "\$sha1: $sha1" +echo "\$ghprbPullTitle: $ghprbPullTitle" # Run pull request tests for t in "${PR_TESTS[@]}"; do @@ -189,6 +190,9 @@ done { # Marks this build is a pull request build. export AMP_JENKINS_PRB=true + if [[ $ghprbPullTitle == *"maven-test"* ]]; then + export AMPLAB_JENKINS_BUILD_TOOL="maven" + fi timeout "${TESTS_TIMEOUT}" ./dev/run-tests test_result="$?" diff --git a/dev/run-tests.py b/dev/run-tests.py index b6d181418f027..29e96b7de15af 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -231,7 +231,7 @@ def exec_maven(mvn_args=()): """Will call Maven in the current directory with the list of mvn_args passed in and returns the subprocess for any further processing""" - run_cmd([os.path.join(SPARK_HOME, "build", "mvn")] + mvn_args) + run_cmd([os.path.join(SPARK_HOME, "build", "mvn")] + ["--force"] + mvn_args) def exec_sbt(sbt_args=()): From a15f53c9ad1859b818ec782fe6d74a2a6d6bbe63 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 15:54:10 -0700 Subject: [PATCH 02/14] Wording change to test-maven --- dev/run-tests-jenkins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 9742f213a33e4..b6341b451b3f1 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -190,7 +190,7 @@ done { # Marks this build is a pull request build. export AMP_JENKINS_PRB=true - if [[ $ghprbPullTitle == *"maven-test"* ]]; then + if [[ $ghprbPullTitle == *"test-maven"* ]]; then export AMPLAB_JENKINS_BUILD_TOOL="maven" fi timeout "${TESTS_TIMEOUT}" ./dev/run-tests From e83b49fd75f0556ca63a257150d59836894a166d Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:03:09 -0700 Subject: [PATCH 03/14] Adding support for SPARK-9547 --- dev/run-tests-jenkins | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index b6341b451b3f1..0e6f422aaa9a4 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -193,6 +193,16 @@ done if [[ $ghprbPullTitle == *"test-maven"* ]]; then export AMPLAB_JENKINS_BUILD_TOOL="maven" fi + if [[ $ghprbPullTitle == *"test-hadoop1.0"* ]]; then + export AMPLAB_JENKINS_BUILD_PROFILE="hadoop1.0" + elif [[ $ghprbPullTitle == *"test-hadoop2.0"* ]]; then + export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.0" + elif [[ $ghprbPullTitle == *"test-hadoop2.2"* ]]; then + export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.2" + elif [[ $ghprbPullTitle == *"test-hadoop2.3"* ]]; then + export AMPLAB_JENKINS_BUILD_PROFILE="hadoop2.3" + fi + timeout "${TESTS_TIMEOUT}" ./dev/run-tests test_result="$?" From 5d15d31bf7f533a4eb92a886c1795f0ebcb0fbc2 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:17:21 -0700 Subject: [PATCH 04/14] Adding randomized zinc ports --- dev/run-tests.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 29e96b7de15af..528dc258ff29c 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -21,6 +21,7 @@ import itertools from optparse import OptionParser import os +import random import re import sys import subprocess @@ -282,17 +283,31 @@ def get_hadoop_profiles(hadoop_version): " are", sbt_maven_hadoop_profiles.keys()) sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) +def get_zinc_port(): + """ + Get a randomized port on which to start Zinc + """ + return random.randrange(3030, 4030) + +def kill_zinc_on_port(zinc_port): + """ + Kill the Zinc process running on the given port, if one exists. + """ + cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " + "| awk '{ print $2; }' | xargs kill") % zinc_port + # TODO: Not sure what happens here if no process exists + run_cmd(cmd) def build_spark_maven(hadoop_version): # Enable all of the profiles for the build: build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags - mvn_goals = ["clean", "package", "-DskipTests"] + zinc_port = get_zinc_port() + mvn_goals = ["clean", "package", "-DskipTests", "-DzincPort=%s" % zinc_port] profiles_and_goals = build_profiles + mvn_goals - print("[info] Building Spark (w/Hive 0.13.1) using Maven with these arguments: ", " ".join(profiles_and_goals)) - exec_maven(profiles_and_goals) + kill_zinc_on_port(zinc_port) def build_spark_sbt(hadoop_version): @@ -331,13 +346,15 @@ def detect_binary_inop_with_mima(): def run_scala_tests_maven(test_profiles): - mvn_test_goals = ["test", "--fail-at-end"] + zinc_port = get_zinc_port() + mvn_test_goals = ["test", "--fail-at-end", "-DzincPort=%s" % zinc_port] profiles_and_goals = test_profiles + mvn_test_goals print("[info] Running Spark tests using Maven with these arguments: ", " ".join(profiles_and_goals)) exec_maven(profiles_and_goals) + kill_zinc_on_port(zinc_port) def run_scala_tests_sbt(test_modules, test_profiles): From 7f1ff75f84011919866041ea71f125af466728d4 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:28:16 -0700 Subject: [PATCH 05/14] Fixing up python lint issues --- dev/run-tests.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 528dc258ff29c..95356fb68a532 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -283,20 +283,25 @@ def get_hadoop_profiles(hadoop_version): " are", sbt_maven_hadoop_profiles.keys()) sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) + def get_zinc_port(): - """ - Get a randomized port on which to start Zinc - """ - return random.randrange(3030, 4030) + """ + Get a randomized port on which to start Zinc + """ + + return random.randrange(3030, 4030) + def kill_zinc_on_port(zinc_port): - """ - Kill the Zinc process running on the given port, if one exists. - """ - cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " - "| awk '{ print $2; }' | xargs kill") % zinc_port - # TODO: Not sure what happens here if no process exists - run_cmd(cmd) + """ + Kill the Zinc process running on the given port, if one exists. + """ + + cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " + "| awk '{ print $2; }' | xargs kill") % zinc_port + # TODO: Not sure what happens here if no process exists + run_cmd(cmd) + def build_spark_maven(hadoop_version): # Enable all of the profiles for the build: From c77f951b1acb0b6554b6bac911b16b8f5c720964 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:29:27 -0700 Subject: [PATCH 06/14] Small style fix --- dev/run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 95356fb68a532..0bb049dc62f7e 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -232,7 +232,7 @@ def exec_maven(mvn_args=()): """Will call Maven in the current directory with the list of mvn_args passed in and returns the subprocess for any further processing""" - run_cmd([os.path.join(SPARK_HOME, "build", "mvn")] + ["--force"] + mvn_args) + run_cmd([os.path.join(SPARK_HOME, "build", "mvn"), "--force"] + mvn_args) def exec_sbt(sbt_args=()): From 69dd9afe387c1db24542df566ae39ccd3ccd90a8 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:40:43 -0700 Subject: [PATCH 07/14] Push Zinc support down into exec_maven --- dev/run-tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 0bb049dc62f7e..7ba333454ecff 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -232,7 +232,12 @@ def exec_maven(mvn_args=()): """Will call Maven in the current directory with the list of mvn_args passed in and returns the subprocess for any further processing""" - run_cmd([os.path.join(SPARK_HOME, "build", "mvn"), "--force"] + mvn_args) + zinc_port = get_zinc_port() + os.environ["ZINC_PORT"] = "%s" % zinc_port + zinc_flag = "-DzincPort=%s" % zinc_port + flags = [os.path.join(SPARK_HOME, "build", "mvn"), "--force", zinc_flag] + run_cmd(flags + mvn_args) + kill_zinc_on_port(zinc_port) def exec_sbt(sbt_args=()): @@ -288,7 +293,6 @@ def get_zinc_port(): """ Get a randomized port on which to start Zinc """ - return random.randrange(3030, 4030) @@ -296,7 +300,6 @@ def kill_zinc_on_port(zinc_port): """ Kill the Zinc process running on the given port, if one exists. """ - cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " "| awk '{ print $2; }' | xargs kill") % zinc_port # TODO: Not sure what happens here if no process exists @@ -306,7 +309,6 @@ def kill_zinc_on_port(zinc_port): def build_spark_maven(hadoop_version): # Enable all of the profiles for the build: build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags - zinc_port = get_zinc_port() mvn_goals = ["clean", "package", "-DskipTests", "-DzincPort=%s" % zinc_port] profiles_and_goals = build_profiles + mvn_goals print("[info] Building Spark (w/Hive 0.13.1) using Maven with these arguments: ", @@ -351,15 +353,13 @@ def detect_binary_inop_with_mima(): def run_scala_tests_maven(test_profiles): - zinc_port = get_zinc_port() - mvn_test_goals = ["test", "--fail-at-end", "-DzincPort=%s" % zinc_port] + mvn_test_goals = ["test", "--fail-at-end"] profiles_and_goals = test_profiles + mvn_test_goals print("[info] Running Spark tests using Maven with these arguments: ", " ".join(profiles_and_goals)) exec_maven(profiles_and_goals) - kill_zinc_on_port(zinc_port) def run_scala_tests_sbt(test_modules, test_profiles): From 0d3944ad91120042193c0bf1e5f4fc9afd3a278b Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 16:49:23 -0700 Subject: [PATCH 08/14] Small fixes --- dev/run-tests.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 7ba333454ecff..ae20e52f25674 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -228,6 +228,23 @@ def build_spark_documentation(): os.chdir(SPARK_HOME) +def get_zinc_port(): + """ + Get a randomized port on which to start Zinc + """ + return random.randrange(3030, 4030) + + +def kill_zinc_on_port(zinc_port): + """ + Kill the Zinc process running on the given port, if one exists. + """ + cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " + "| awk '{ print $2; }' | xargs kill") % zinc_port + # TODO: Not sure what happens here if no process exists + run_cmd(cmd) + + def exec_maven(mvn_args=()): """Will call Maven in the current directory with the list of mvn_args passed in and returns the subprocess for any further processing""" @@ -289,32 +306,14 @@ def get_hadoop_profiles(hadoop_version): sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) -def get_zinc_port(): - """ - Get a randomized port on which to start Zinc - """ - return random.randrange(3030, 4030) - - -def kill_zinc_on_port(zinc_port): - """ - Kill the Zinc process running on the given port, if one exists. - """ - cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " - "| awk '{ print $2; }' | xargs kill") % zinc_port - # TODO: Not sure what happens here if no process exists - run_cmd(cmd) - - def build_spark_maven(hadoop_version): # Enable all of the profiles for the build: build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags - mvn_goals = ["clean", "package", "-DskipTests", "-DzincPort=%s" % zinc_port] + mvn_goals = ["clean", "package", "-DskipTests"] profiles_and_goals = build_profiles + mvn_goals print("[info] Building Spark (w/Hive 0.13.1) using Maven with these arguments: ", " ".join(profiles_and_goals)) exec_maven(profiles_and_goals) - kill_zinc_on_port(zinc_port) def build_spark_sbt(hadoop_version): From 05d7720bd1fafb2f8acecea1e9be1cccf9b2fdac Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 17:02:45 -0700 Subject: [PATCH 09/14] Bug fix to kill command --- dev/run-tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index ae20e52f25674..c4d369254bd9a 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -241,8 +241,7 @@ def kill_zinc_on_port(zinc_port): """ cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN " "| awk '{ print $2; }' | xargs kill") % zinc_port - # TODO: Not sure what happens here if no process exists - run_cmd(cmd) + subprocess.check_call(cmd, shell=True) def exec_maven(mvn_args=()): From 3d36d0a4172a6d3379be196917dda1d113a74bba Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 17:25:03 -0700 Subject: [PATCH 10/14] Temporarily disabling mima --- dev/run-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index c4d369254bd9a..f3f6db6fb2bd6 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -512,7 +512,7 @@ def main(): build_apache_spark(build_tool, hadoop_version) # backwards compatibility checks - detect_binary_inop_with_mima() + # FIXME: TEMPORARILY DISABLED detect_binary_inop_with_mima() # run the test suites run_scala_tests(build_tool, hadoop_version, test_modules) From 2e9edcc5163456f814e7d9a707d3ad86f775b4af Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 2 Aug 2015 22:09:12 -0700 Subject: [PATCH 11/14] Increasing timeout --- dev/run-tests-jenkins | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 0e6f422aaa9a4..55bf89d535556 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -48,8 +48,8 @@ COMMIT_URL="https://github.com/apache/spark/commit/${ghprbActualCommit}" SHORT_COMMIT_HASH="${ghprbActualCommit:0:7}" # format: http://linux.die.net/man/1/timeout -# must be less than the timeout configured on Jenkins (currently 180m) -TESTS_TIMEOUT="175m" +# must be less than the timeout configured on Jenkins (currently 205m) +TESTS_TIMEOUT="200m" # Array to capture all tests to run on the pull request. These tests are held under the #+ dev/tests/ directory. From a938de344b60d347de9a2fa812cb4909a6341330 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Mon, 3 Aug 2015 18:17:29 -0700 Subject: [PATCH 12/14] Only do compatibility tests in sbt --- dev/run-tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 5ea0213335e24..c9265e17d131a 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -513,7 +513,9 @@ def main(): build_apache_spark(build_tool, hadoop_version) # backwards compatibility checks - # FIXME: TEMPORARILY DISABLED detect_binary_inop_with_mima() + if build_tool == "sbt": + # Note: compatiblity tests only supported in sbt for now + detect_binary_inop_with_mima() # run the test suites run_scala_tests(build_tool, hadoop_version, test_modules) From 603f891303483954cc65f27bd9b39469dc498f7c Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Mon, 3 Aug 2015 18:18:41 -0700 Subject: [PATCH 13/14] removing unncessary spacing changes --- dev/run-tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/run-tests.py b/dev/run-tests.py index c9265e17d131a..c757cab401c87 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -311,8 +311,10 @@ def build_spark_maven(hadoop_version): build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags mvn_goals = ["clean", "package", "-DskipTests"] profiles_and_goals = build_profiles + mvn_goals + print("[info] Building Spark (w/Hive 1.2.1) using Maven with these arguments: ", " ".join(profiles_and_goals)) + exec_maven(profiles_and_goals) From 87d4d1934b087a6a73471afef10b5d29569f7141 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Mon, 3 Aug 2015 19:52:39 -0700 Subject: [PATCH 14/14] Whitepsace issues --- dev/run-tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index c757cab401c87..f12183b7a2bef 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -311,7 +311,7 @@ def build_spark_maven(hadoop_version): build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags mvn_goals = ["clean", "package", "-DskipTests"] profiles_and_goals = build_profiles + mvn_goals - + print("[info] Building Spark (w/Hive 1.2.1) using Maven with these arguments: ", " ".join(profiles_and_goals)) @@ -516,8 +516,8 @@ def main(): # backwards compatibility checks if build_tool == "sbt": - # Note: compatiblity tests only supported in sbt for now - detect_binary_inop_with_mima() + # Note: compatiblity tests only supported in sbt for now + detect_binary_inop_with_mima() # run the test suites run_scala_tests(build_tool, hadoop_version, test_modules)