From 28e9fb417b50c7408b8308c590edf781238eaffa Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 17 Jan 2021 18:36:44 -0800 Subject: [PATCH 1/4] [SPARK-31168][BUILD] Upgrade Scala to 2.12.13 --- dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 6 +++--- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 6 +++--- docs/_config.yml | 2 +- pom.xml | 2 +- project/SparkBuild.scala | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 86d2c3ab2b159..963a0f2417418 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -212,10 +212,10 @@ protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.1//py4j-0.10.9.1.jar pyrolite/4.30//pyrolite-4.30.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar +scala-compiler/2.12.13//scala-compiler-2.12.13.jar +scala-library/2.12.13//scala-library-2.12.13.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar +scala-reflect/2.12.13//scala-reflect-2.12.13.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.9.0//shims-0.9.0.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 7d3876c347178..60911d1f7a2f6 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -182,10 +182,10 @@ protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.1//py4j-0.10.9.1.jar pyrolite/4.30//pyrolite-4.30.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar +scala-compiler/2.12.13//scala-compiler-2.12.13.jar +scala-library/2.12.13//scala-library-2.12.13.jar scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar +scala-reflect/2.12.13//scala-reflect-2.12.13.jar scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar shims/0.9.0//shims-0.9.0.jar diff --git a/docs/_config.yml b/docs/_config.yml index a8d42e483d17d..04426471ba264 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -22,7 +22,7 @@ include: SPARK_VERSION: 3.2.0-SNAPSHOT SPARK_VERSION_SHORT: 3.2.0 SCALA_BINARY_VERSION: "2.12" -SCALA_VERSION: "2.12.10" +SCALA_VERSION: "2.12.13" MESOS_VERSION: 1.0.0 SPARK_ISSUE_TRACKER_URL: https://issues.apache.org/jira/browse/SPARK SPARK_GITHUB_URL: https://github.com/apache/spark diff --git a/pom.xml b/pom.xml index 64b09b45824c0..5db0bb32688b5 100644 --- a/pom.xml +++ b/pom.xml @@ -163,7 +163,7 @@ 3.4.1 3.2.2 - 2.12.10 + 2.12.13 2.12 2.0.0 --test diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 668701be0ae98..817ab90bee6dc 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -206,7 +206,7 @@ object SparkBuild extends PomBuild { lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq( libraryDependencies ++= { if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) { - val silencerVersion = if (scalaBinaryVersion.value == "2.13") "1.7.1" else "1.6.0" + val silencerVersion = "1.7.1" Seq( "org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0", compilerPlugin("com.github.ghik" % "silencer-plugin" % silencerVersion cross CrossVersion.full), From fc235a7394038afe23c94906c1a524d233da9697 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 17 Jan 2021 21:29:40 -0800 Subject: [PATCH 2/4] Upgrade kafka to 2.7.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5db0bb32688b5..1ee03f2b064c6 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 2.3 - 2.6.0 + 2.7.0 10.14.2.0 1.10.1 From 2004207111d0ea0c92b1bef65c8620135016e1d4 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 17 Jan 2021 22:24:17 -0800 Subject: [PATCH 3/4] Update getLogManager usage --- .../scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala index 43ed4a8378a8c..6a4990e1cd11e 100644 --- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala +++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala @@ -574,11 +574,11 @@ class KafkaTestUtils( s"topic $topic still exists in the replica manager") // ensure that logs from all replicas are deleted if delete topic is marked successful assert(servers.forall(server => topicAndPartitions.forall(tp => - server.getLogManager().getLog(tp).isEmpty)), + server.getLogManager.getLog(tp).isEmpty)), s"topic $topic still exists in log manager") // ensure that topic is removed from all cleaner offsets assert(servers.forall(server => topicAndPartitions.forall { tp => - val checkpoints = server.getLogManager().liveLogDirs.map { logDir => + val checkpoints = server.getLogManager.liveLogDirs.map { logDir => new OffsetCheckpointFile(new File(logDir, "cleaner-offset-checkpoint")).read() } checkpoints.forall(checkpointsPerLogDir => !checkpointsPerLogDir.contains(tp)) From 39a4aa798ff0d9afd7891ac4e74a5da28aec1357 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 18 Jan 2021 12:05:04 -0800 Subject: [PATCH 4/4] Upgrade silencer --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ee03f2b064c6..a08e4b6cfd14e 100644 --- a/pom.xml +++ b/pom.xml @@ -2564,7 +2564,7 @@ com.github.ghik silencer-plugin_${scala.version} - 1.6.0 + 1.7.1