From 9fb0fb0efe075ae4ab9ecc0feb35f47f1aa7f256 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 19 Sep 2023 19:44:45 +0800 Subject: [PATCH 1/9] for maven --- assembly/pom.xml | 4 +- common/kvstore/pom.xml | 4 +- common/network-common/pom.xml | 4 +- common/network-shuffle/pom.xml | 4 +- common/network-yarn/pom.xml | 4 +- common/sketch/pom.xml | 4 +- common/tags/pom.xml | 4 +- common/unsafe/pom.xml | 4 +- common/utils/pom.xml | 4 +- connector/avro/pom.xml | 8 +- connector/connect/client/jvm/pom.xml | 4 +- connector/connect/common/pom.xml | 4 +- connector/connect/server/pom.xml | 8 +- connector/docker-integration-tests/pom.xml | 4 +- connector/kafka-0-10-assembly/pom.xml | 4 +- connector/kafka-0-10-sql/pom.xml | 8 +- connector/kafka-0-10-token-provider/pom.xml | 4 +- connector/kafka-0-10/pom.xml | 8 +- connector/kinesis-asl-assembly/pom.xml | 4 +- connector/kinesis-asl/pom.xml | 4 +- connector/protobuf/pom.xml | 8 +- connector/spark-ganglia-lgpl/pom.xml | 4 +- core/pom.xml | 8 +- dev/mima | 8 +- docs/_plugins/copy_api_dirs.rb | 14 +- examples/pom.xml | 4 +- graphx/pom.xml | 4 +- hadoop-cloud/pom.xml | 4 +- launcher/pom.xml | 4 +- mllib-local/pom.xml | 4 +- mllib/pom.xml | 8 +- pom.xml | 148 +++++++----------- repl/pom.xml | 4 +- resource-managers/kubernetes/core/pom.xml | 4 +- .../kubernetes/integration-tests/pom.xml | 4 +- resource-managers/mesos/pom.xml | 4 +- resource-managers/yarn/pom.xml | 4 +- sql/api/pom.xml | 4 +- sql/catalyst/pom.xml | 8 +- sql/core/pom.xml | 8 +- sql/hive-thriftserver/pom.xml | 8 +- sql/hive/pom.xml | 8 +- streaming/pom.xml | 8 +- tools/pom.xml | 4 +- 44 files changed, 171 insertions(+), 211 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 69952a7ccfa02..020ef80c4ad3e 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-assembly_2.12 + spark-assembly_2.13 Spark Project Assembly https://spark.apache.org/ pom diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 6a4d029562af0..54b7f401cc4f0 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-kvstore_2.12 + spark-kvstore_2.13 jar Spark Project Local DB https://spark.apache.org/ diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 1823edbe0f53d..a37f64a962fbc 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-network-common_2.12 + spark-network-common_2.13 jar Spark Project Networking https://spark.apache.org/ diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 8fccc855a53fa..0f7036ef746cc 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-network-shuffle_2.12 + spark-network-shuffle_2.13 jar Spark Project Shuffle Streaming Service https://spark.apache.org/ diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 57e18f7a43dca..5661a1d59a6b9 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-network-yarn_2.12 + spark-network-yarn_2.13 jar Spark Project YARN Shuffle Service https://spark.apache.org/ diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 3d24494e543f7..e3f3d4127b717 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-sketch_2.12 + spark-sketch_2.13 jar Spark Project Sketch https://spark.apache.org/ diff --git a/common/tags/pom.xml b/common/tags/pom.xml index cdeb862abd5f9..1894b6f4acb76 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-tags_2.12 + spark-tags_2.13 jar Spark Project Tags https://spark.apache.org/ diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 284d0176afdde..4d23c9149bb7d 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-unsafe_2.12 + spark-unsafe_2.13 jar Spark Project Unsafe https://spark.apache.org/ diff --git a/common/utils/pom.xml b/common/utils/pom.xml index 2f2fee0cf41e8..37d1ea48d972a 100644 --- a/common/utils/pom.xml +++ b/common/utils/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-common-utils_2.12 + spark-common-utils_2.13 jar Spark Project Common Utils https://spark.apache.org/ diff --git a/connector/avro/pom.xml b/connector/avro/pom.xml index d614ccf038466..bde5d5de6eae9 100644 --- a/connector/avro/pom.xml +++ b/connector/avro/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-avro_2.12 + spark-avro_2.13 avro @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.tukaani xz diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 8cb6758ec9f1f..9ca66b5c29ca2 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../../../pom.xml - spark-connect-client-jvm_2.12 + spark-connect-client-jvm_2.13 jar Spark Project Connect Client https://spark.apache.org/ diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index 5fd6c88502552..517f4c7c9a0b6 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../../pom.xml - spark-connect-common_2.12 + spark-connect-common_2.13 jar Spark Project Connect Common https://spark.apache.org/ diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index e98b8da8e5c07..291da8d870613 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../../pom.xml - spark-connect_2.12 + spark-connect_2.13 jar Spark Project Connect Server https://spark.apache.org/ @@ -152,12 +152,12 @@ - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.guava guava diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml index a9c066ede2d80..4308abbbe2016 100644 --- a/connector/docker-integration-tests/pom.xml +++ b/connector/docker-integration-tests/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-docker-integration-tests_2.12 + spark-docker-integration-tests_2.13 jar Spark Project Docker Integration Tests https://spark.apache.org/ diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index bcc3503672e67..b2fcbdf8eca7d 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10-assembly_2.12 + spark-streaming-kafka-0-10-assembly_2.13 jar Spark Integration for Kafka 0.10 Assembly https://spark.apache.org/ diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index 9d7ec2c5491b0..56f0b874b2b9e 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-sql-kafka-0-10_2.12 + spark-sql-kafka-0-10_2.13 sql-kafka-0-10 @@ -74,12 +74,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.kafka kafka-clients diff --git a/connector/kafka-0-10-token-provider/pom.xml b/connector/kafka-0-10-token-provider/pom.xml index c53091dba5d0f..2b2707b9da320 100644 --- a/connector/kafka-0-10-token-provider/pom.xml +++ b/connector/kafka-0-10-token-provider/pom.xml @@ -20,13 +20,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml org.apache.spark - spark-token-provider-kafka-0-10_2.12 + spark-token-provider-kafka-0-10_2.13 token-provider-kafka-0-10 diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml index 8f7ef1541765c..4c76c8a9fe46a 100644 --- a/connector/kafka-0-10/pom.xml +++ b/connector/kafka-0-10/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-streaming-kafka-0-10_2.12 + spark-streaming-kafka-0-10_2.13 streaming-kafka-0-10 @@ -59,12 +59,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.kafka kafka-clients diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml index 2232f581ceabf..ecd7d13dcba94 100644 --- a/connector/kinesis-asl-assembly/pom.xml +++ b/connector/kinesis-asl-assembly/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl-assembly_2.12 + spark-streaming-kinesis-asl-assembly_2.13 jar Spark Project Kinesis Assembly https://spark.apache.org/ diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml index 881098f330fd5..9a7f40443bbc9 100644 --- a/connector/kinesis-asl/pom.xml +++ b/connector/kinesis-asl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-streaming-kinesis-asl_2.12 + spark-streaming-kinesis-asl_2.13 jar Spark Kinesis Integration diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index 26b5b601bccd8..7854a9c59dcc2 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-protobuf_2.12 + spark-protobuf_2.13 protobuf @@ -70,12 +70,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.protobuf protobuf-java diff --git a/connector/spark-ganglia-lgpl/pom.xml b/connector/spark-ganglia-lgpl/pom.xml index 2f8e3ba015e9a..4c9a0c7347e64 100644 --- a/connector/spark-ganglia-lgpl/pom.xml +++ b/connector/spark-ganglia-lgpl/pom.xml @@ -19,13 +19,13 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-ganglia-lgpl_2.12 + spark-ganglia-lgpl_2.13 jar Spark Ganglia Integration diff --git a/core/pom.xml b/core/pom.xml index 24f60c6c83814..8e2f42225317f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-core_2.12 + spark-core_2.13 jar Spark Project Core https://spark.apache.org/ @@ -35,12 +35,12 @@ - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.avro avro diff --git a/dev/mima b/dev/mima index 859301b4d669c..17e9abe4bfb92 100755 --- a/dev/mima +++ b/dev/mima @@ -24,9 +24,9 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} -TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" -OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" +SPARK_PROFILES=${1:-"-Pscala-2.13 -Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} +TOOLS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false "export tools/fullClasspath" | grep jar | tail -n1)" +OLD_DEPS_CLASSPATH="$(build/sbt -Pscala-2.13 -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | grep jar | tail -n1)" rm -f .generated-mima* @@ -42,7 +42,7 @@ $JAVA_CMD \ -cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \ org.apache.spark.tools.GenerateMIMAIgnore -echo -e "q\n" | build/sbt -mem 5632 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" +echo -e "q\n" | build/sbt -Pscala-2.13 -mem 5632 -DcopyDependencies=false "$@" mimaReportBinaryIssues | grep -v -e "info.*Resolving" ret_val=$? if [ $ret_val != 0 ]; then diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 28d5e0d82c93a..9cb073ef1e00c 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -26,8 +26,8 @@ curr_dir = pwd cd("..") - puts "Running 'build/sbt -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." - system("build/sbt -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") + puts "Running 'build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 -Pkinesis-asl clean compile unidoc") || raise("Unidoc generation failed") puts "Moving back into docs dir." cd("docs") @@ -37,7 +37,7 @@ # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.12/unidoc" + source = "../target/scala-2.13/unidoc" dest = "api/scala" puts "Making directory " + dest @@ -119,8 +119,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("PySpark doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("PySpark doc generation failed") puts "Moving back into docs dir." cd("docs") @@ -165,8 +165,8 @@ puts "Moving to project root and building API docs." cd("..") - puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..." - system("build/sbt clean package -Phive") || raise("SQL doc generation failed") + puts "Running 'build/sbt -Pscala-2.13 clean package -Phive' from " + pwd + "; this may take a few minutes..." + system("build/sbt -Pscala-2.13 clean package -Phive") || raise("SQL doc generation failed") puts "Moving back into docs dir." cd("docs") diff --git a/examples/pom.xml b/examples/pom.xml index c5644f6a08950..9470f13ecfc2c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-examples_2.12 + spark-examples_2.13 jar Spark Project Examples https://spark.apache.org/ diff --git a/graphx/pom.xml b/graphx/pom.xml index 3771a274082cd..ce29c1845422a 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-graphx_2.12 + spark-graphx_2.13 graphx diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index b27df3597eb13..934ff29be4060 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-hadoop-cloud_2.12 + spark-hadoop-cloud_2.13 jar Spark Project Hadoop Cloud Integration diff --git a/launcher/pom.xml b/launcher/pom.xml index d87f7bd8fef5c..c47244ff887a6 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-launcher_2.12 + spark-launcher_2.13 jar Spark Project Launcher https://spark.apache.org/ diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index fe8ec5721ea7e..408aec1ff276b 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-mllib-local_2.12 + spark-mllib-local_2.13 mllib-local diff --git a/mllib/pom.xml b/mllib/pom.xml index 0b5292d7c63b5..70c116846f4bf 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-mllib_2.12 + spark-mllib_2.13 mllib @@ -91,12 +91,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.scalanlp breeze_${scala.binary.version} diff --git a/pom.xml b/pom.xml index 971cb07ea40ea..394ddaad3794a 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 18 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT pom Spark Project Parent POM @@ -172,8 +172,8 @@ 3.2.2 4.4 - 2.12.18 - 2.12 + 2.13.11 + 2.13 2.2.0 4.7.1 @@ -440,13 +440,13 @@ ${project.version} test-jar - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} 1.0.4 - --> + com.twitter chill_${scala.binary.version} @@ -1110,7 +1110,7 @@ org.scala-lang.modules - scala-xml_2.12 + scala-xml_2.13 @@ -2934,9 +2934,53 @@ -feature -explaintypes -target:jvm-1.8 - -Xfatal-warnings - -Ywarn-unused:imports - -P:silencer:globalFilters=.*deprecated.* + -Wconf:cat=deprecation:wv,any:e + -Wunused:imports + + -Wconf:cat=scaladoc:wv + -Wconf:cat=lint-multiarg-infix:wv + -Wconf:cat=other-nullary-override:wv + -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv + -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv + + -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s + -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s + -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s + -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s + -Wconf:msg=method without a parameter list overrides a method with a single empty one:s + + -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e + + -Wconf:cat=unchecked&msg=outer reference:s + -Wconf:cat=unchecked&msg=eliminated by erasure:s + -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s + + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBase.scala:s + -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBaseOps.scala:s + + -Wconf:msg=Implicit definition should have explicit type:s -Xss128m @@ -2952,13 +2996,6 @@ ${java.version} -Xlint:all,-serial,-path,-try - - - com.github.ghik - silencer-plugin_${scala.version} - 1.7.13 - - @@ -3611,97 +3648,20 @@ - scala-2.12 + scala-2.13 - 2.12.18 - - - - - - - - - - - scala-2.13 - 2.13.11 - 2.13 - - net.alchim31.maven - scala-maven-plugin - - - -unchecked - -deprecation - -feature - -explaintypes - -target:jvm-1.8 - -Wconf:cat=deprecation:wv,any:e - -Wunused:imports - - -Wconf:cat=scaladoc:wv - -Wconf:cat=lint-multiarg-infix:wv - -Wconf:cat=other-nullary-override:wv - -Wconf:cat=other-match-analysis&site=org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction.catalogFunction:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.streaming.util.FileBasedWriteAheadLog.readAll.readFile:wv - -Wconf:cat=other-pure-statement&site=org.apache.spark.scheduler.OutputCommitCoordinatorSuite.<local OutputCommitCoordinatorSuite>.futureAction:wv - - -Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s - -Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s - -Wconf:msg=Auto-application to \`\(\)\` is deprecated:s - -Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s - -Wconf:msg=method without a parameter list overrides a method with a single empty one:s - - -Wconf:cat=deprecation&msg=procedure syntax is deprecated:e - - -Wconf:cat=unchecked&msg=outer reference:s - -Wconf:cat=unchecked&msg=eliminated by erasure:s - -Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s - - -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBase.scala:s - -Wconf:cat=unused-imports&src=org\/apache\/spark\/graphx\/impl\/VertexPartitionBaseOps.scala:s - - -Wconf:msg=Implicit definition should have explicit type:s - - - - - - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.scalacheck scalacheck_${scala.binary.version} diff --git a/sql/core/pom.xml b/sql/core/pom.xml index bf3caf58fe276..914b0c63db135 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-sql_2.12 + spark-sql_2.13 jar Spark Project SQL https://spark.apache.org/ @@ -89,12 +89,12 @@ test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + org.apache.orc orc-core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 76a1037f1cbac..4d7b6696b0c27 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-hive-thriftserver_2.12 + spark-hive-thriftserver_2.13 jar Spark Project Hive Thrift Server https://spark.apache.org/ @@ -61,12 +61,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + com.google.guava guava diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 7a12c6222eb4d..adcf0f55dcc1c 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,12 +21,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../../pom.xml - spark-hive_2.12 + spark-hive_2.13 jar Spark Project Hive https://spark.apache.org/ @@ -79,12 +79,12 @@ test-jar test - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + ${hive.group} hive-common diff --git a/streaming/pom.xml b/streaming/pom.xml index b36289e4e4945..673f719f2cb9b 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,12 +20,12 @@ 4.0.0 org.apache.spark - spark-parent_2.12 + spark-parent_2.13 4.0.0-SNAPSHOT ../pom.xml - spark-streaming_2.12 + spark-streaming_2.13 streaming @@ -50,12 +50,12 @@ org.apache.spark spark-tags_${scala.binary.version} - org.scala-lang.modules scala-parallel-collections_${scala.binary.version} - --> + - - org.codehaus.mojo - build-helper-maven-plugin - - - add-scala-sources - generate-sources - - add-source - - - - src/main/scala-${scala.binary.version} - - - - - add-scala-test-sources - generate-test-sources - - add-test-source - - - - src/test/scala-${scala.binary.version} - - - - - - diff --git a/sql/api/pom.xml b/sql/api/pom.xml index 01dc503784d51..bcf01bbe0cd9a 100644 --- a/sql/api/pom.xml +++ b/sql/api/pom.xml @@ -81,24 +81,6 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources - generate-sources - - add-source - - - - src/main/scala-${scala.binary.version} - - - - - org.antlr antlr4-maven-plugin diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 1a2e97141ad52..48702f69a8a50 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -154,24 +154,6 @@ -ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources - generate-sources - - add-source - - - - src/main/scala-${scala.binary.version} - - - - - diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 914b0c63db135..ab7d1c07ee28e 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -263,18 +263,6 @@ org.codehaus.mojo build-helper-maven-plugin - - add-sources - generate-sources - - add-source - - - - src/main/scala-${scala.binary.version} - - - add-scala-test-sources generate-test-sources From f5b18233a58de7d7b73729d8859e75676be0857c Mon Sep 17 00:00:00 2001 From: YangJie Date: Fri, 22 Sep 2023 11:37:35 +0800 Subject: [PATCH 9/9] Update docs/building-spark.md Co-authored-by: Cheng Pan --- docs/building-spark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/building-spark.md b/docs/building-spark.md index 48edf1155620c..b7d1bacea3cc7 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -28,7 +28,7 @@ license: | The Maven-based build is the build of reference for Apache Spark. Building Spark using Maven requires Maven 3.9.4 and Java 8/11/17. -Spark requires 2.13; support for Scala 2.12 was removed in Spark 4.0.0. +Spark requires Scala 2.13; support for Scala 2.12 was removed in Spark 4.0.0. ### Setting up Maven's Memory Usage