From 7811091bc02ac49f5ff334d25dcedbe165068ae9 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Tue, 17 Mar 2020 03:59:44 +0800 Subject: [PATCH 1/4] Upgrade Scala to 2.12.11 --- dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 6 +++--- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 6 +++--- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 6 +++--- docs/_config.yml | 2 +- external/docker/spark-test/base/Dockerfile | 2 +- pom.xml | 2 +- .../org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2 index 6ab937ff40652..d75e086aa558c 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2 @@ -177,10 +177,10 @@ protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9//py4j-0.10.9.jar pyrolite/4.30//pyrolite-4.30.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar +scala-compiler/2.12.11//scala-compiler-2.12.11.jar +scala-library/2.12.11//scala-library-2.12.11.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar +scala-reflect/2.12.11//scala-reflect-2.12.11.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.7.45//shims-0.7.45.jar diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 6f563817ab2b6..c6c02dbb8e590 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -191,10 +191,10 @@ protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9//py4j-0.10.9.jar pyrolite/4.30//pyrolite-4.30.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar +scala-compiler/2.12.11//scala-compiler-2.12.11.jar +scala-library/2.12.11//scala-library-2.12.11.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar +scala-reflect/2.12.11//scala-reflect-2.12.11.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.7.45//shims-0.7.45.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 90cbd023d7664..1ce175425bc0a 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -207,10 +207,10 @@ py4j/0.10.9//py4j-0.10.9.jar pyrolite/4.30//pyrolite-4.30.jar re2j/1.1//re2j-1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar +scala-compiler/2.12.11//scala-compiler-2.12.11.jar +scala-library/2.12.11//scala-library-2.12.11.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar +scala-reflect/2.12.11//scala-reflect-2.12.11.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.7.45//shims-0.7.45.jar diff --git a/docs/_config.yml b/docs/_config.yml index f82394ed63694..a9e8130e3eef9 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -17,7 +17,7 @@ include: SPARK_VERSION: 3.1.0-SNAPSHOT SPARK_VERSION_SHORT: 3.1.0 SCALA_BINARY_VERSION: "2.12" -SCALA_VERSION: "2.12.10" +SCALA_VERSION: "2.12.1" MESOS_VERSION: 1.0.0 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK SPARK_GITHUB_URL: https://github.com/apache/spark diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile index 5bec5d3f16548..2968def8133c8 100644 --- a/external/docker/spark-test/base/Dockerfile +++ b/external/docker/spark-test/base/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && \ apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \ rm -rf /var/lib/apt/lists/* -ENV SCALA_VERSION 2.12.10 +ENV SCALA_VERSION 2.12.11 ENV CDH_VERSION cdh4 ENV SCALA_HOME /opt/scala-$SCALA_VERSION ENV SPARK_HOME /opt/spark diff --git a/pom.xml b/pom.xml index c90ac680beffb..b9288f1809b0b 100644 --- a/pom.xml +++ b/pom.xml @@ -160,7 +160,7 @@ 3.4.1 3.2.2 - 2.12.10 + 2.12.11 2.12 --test diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 1f7634bafa420..8432074e2d1e7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -72,7 +72,7 @@ object ScalaReflection extends ScalaReflection { /** * Synchronize to prevent concurrent usage of `<:<` operator. * This operator is not thread safe in any current version of scala; i.e. - * (2.11.12, 2.12.10, 2.13.0-M5). + * (2.11.12, 2.12.11, 2.13.0-M5). * * See https://github.com/scala/bug/issues/10766 */ From fa21f57c1afff4830133b33c7b337beff07011bd Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Tue, 17 Mar 2020 09:54:41 +0800 Subject: [PATCH 2/4] Fix typo --- docs/_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_config.yml b/docs/_config.yml index a9e8130e3eef9..89a1eb60850fd 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -17,7 +17,7 @@ include: SPARK_VERSION: 3.1.0-SNAPSHOT SPARK_VERSION_SHORT: 3.1.0 SCALA_BINARY_VERSION: "2.12" -SCALA_VERSION: "2.12.1" +SCALA_VERSION: "2.12.11" MESOS_VERSION: 1.0.0 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK SPARK_GITHUB_URL: https://github.com/apache/spark From 678e5d6b4d17f5f97750b433f939411241059987 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Wed, 18 Mar 2020 22:33:23 +0800 Subject: [PATCH 3/4] git commit --allow-empty -m "Trigger GithubAction" From f1d8e389708eb4cb4963228e436a16754d22aa47 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 20 Mar 2020 15:59:22 +0800 Subject: [PATCH 4/4] fail-fast: false && tests_timeout = "600m" --- .github/workflows/master.yml | 1 + dev/run-tests-jenkins.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 632a0f1aa91fd..24d33fa4ad37a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -13,6 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: java: [ '1.8', '11' ] hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ] diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py index 72e32d4e16e14..b9298e317e3a6 100755 --- a/dev/run-tests-jenkins.py +++ b/dev/run-tests-jenkins.py @@ -198,7 +198,7 @@ def main(): # format: http://linux.die.net/man/1/timeout # must be less than the timeout configured on Jenkins. Usually Jenkins's timeout is higher # then this. Please consult with the build manager or a committer when it should be increased. - tests_timeout = "400m" + tests_timeout = "600m" # Array to capture all test names to run on the pull request. These tests are represented # by their file equivalents in the dev/tests/ directory.