From 995ef1a9c6de539df2488c5f9ebfea45f579e2e1 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 20 May 2019 11:27:08 +0900 Subject: [PATCH 1/3] Exclude 'hive-thriftserver' in modules to test for hadoop3.2 for now --- dev/run-tests.py | 9 ++++++++- dev/sparktestsupport/modules.py | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 5211aea03ea49..592fbee823eea 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -112,6 +112,7 @@ def determine_modules_to_test(changed_modules): ['graphx', 'examples'] >>> x = [x.name for x in determine_modules_to_test([modules.sql])] >>> x # doctest: +NORMALIZE_WHITESPACE + ... # doctest: +SKIP ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver', 'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml'] """ @@ -122,9 +123,15 @@ def determine_modules_to_test(changed_modules): # If we need to run all of the tests, then we should short-circuit and return 'root' if modules.root in modules_to_test: return [modules.root] - return toposort_flatten( + changed_modules = toposort_flatten( {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True) + # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it + if modules.hadoop_version == "hadoop3.2": + changed_modules = [m for m in changed_modules if m.name != "hive-thriftserver"] + + return changed_modules + def determine_tags_to_exclude(changed_modules): tags = [] diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 812c2efae9932..7fe3b1be75d93 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -21,12 +21,6 @@ import re import os -if os.environ.get("AMPLAB_JENKINS"): - hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7") -else: - hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7") -print("[info] Choosing supported modules with Hadoop profile", hadoop_version) - all_modules = [] @@ -80,11 +74,7 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags= self.dependent_modules = set() for dep in dependencies: dep.dependent_modules.add(self) - # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it - if name == "hive-thriftserver" and hadoop_version == "hadoop3.2": - print("[info] Skip unsupported module:", name) - else: - all_modules.append(self) + all_modules.append(self) def contains_file(self, filename): return any(re.match(p, filename) for p in self.source_file_prefixes) @@ -568,6 +558,15 @@ def __hash__(self): ] ) +# TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it +if os.environ.get("AMPLAB_JENKINS"): + hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7") +else: + hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7") +if hadoop_version == "hadoop3.2": + print("[info] Skip unsupported module:", "hive-thriftserver") + all_modules = [m for m in all_modules if m.name != "hive-thriftserver"] + # The root module is a dummy module which is used to run all of the tests. # No other modules should directly depend on this module. root = Module( From fd05eaf34e86afdf1fff9d1d0ab09a01a3c95b33 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 20 May 2019 11:40:36 +0900 Subject: [PATCH 2/3] Change in SQL to check the module discovery --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index b4c68a7014111..344ff8cdbb66d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1770,7 +1770,7 @@ object SQLConf { } /** - * A class that enables the setting and getting of mutable config parameters/hints. + * A class that enables the setting and getting of mutable config parameters/hints * * In the presence of a SQLContext, these can be set and queried by passing SET commands * into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can From 92abff88ea626812c2b6e161122bc4d0766be4a7 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Mon, 20 May 2019 11:44:44 +0900 Subject: [PATCH 3/3] Revert "Change in SQL to check the module discovery" This reverts commit fd05eaf34e86afdf1fff9d1d0ab09a01a3c95b33. --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 344ff8cdbb66d..b4c68a7014111 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1770,7 +1770,7 @@ object SQLConf { } /** - * A class that enables the setting and getting of mutable config parameters/hints + * A class that enables the setting and getting of mutable config parameters/hints. * * In the presence of a SQLContext, these can be set and queried by passing SET commands * into Spark SQL's query functions (i.e. sql()). Otherwise, users of this class can