From 4079d7a67b88f6312e50b40af64780ceea6c75ef Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 11:27:14 +0800
Subject: [PATCH 01/14] init

---
 .github/workflows/benchmark.yml               |  4 +-
 .github/workflows/build_and_test.yml          | 18 +++----
 .github/workflows/build_ansi.yml              |  4 +-
 .github/workflows/build_coverage.yml          |  4 +-
 .github/workflows/build_java11.yml            | 49 -------------------
 .../workflows/build_rockdb_as_ui_backend.yml  |  4 +-
 .github/workflows/build_scala213.yml          |  4 +-
 .github/workflows/publish_snapshot.yml        |  7 +--
 docs/building-spark.md                        |  2 +-
 docs/index.md                                 |  5 +-
 pom.xml                                       |  6 +--
 11 files changed, 27 insertions(+), 80 deletions(-)
 delete mode 100644 .github/workflows/build_java11.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 9e5b0f424838a..43c6e39e15cce 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -27,9 +27,9 @@ on:
         required: true
         default: '*'
       jdk:
-        description: 'JDK version: 8, 11, 17 or 21-ea'
+        description: 'JDK version: 17 or 21-ea'
         required: true
-        default: '8'
+        default: '17'
       scala:
         description: 'Scala version: 2.12 or 2.13'
         required: true
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index e18c54562f006..7104a29108940 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -25,7 +25,7 @@ on:
       java:
         required: false
         type: string
-        default: 8
+        default: 17
       branch:
         description: Branch to run the build against
         required: false
@@ -650,11 +650,11 @@ jobs:
         if [ -f ./dev/free_disk_space_container ]; then
           ./dev/free_disk_space_container
         fi
-    - name: Install Java 8
+    - name: Install Java 17
       uses: actions/setup-java@v3
       with:
         distribution: temurin
-        java-version: 8
+        java-version: 17
     - name: License test
       run: ./dev/check-license
     - name: Dependencies test
@@ -865,11 +865,11 @@ jobs:
         key: scala-213-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           scala-213-coursier-
-    - name: Install Java 8
+    - name: Install Java 17
       uses: actions/setup-java@v3
       with:
         distribution: temurin
-        java-version: 8
+        java-version: 17
     - name: Build with SBT
       run: |
         ./dev/change-scala-version.sh 2.13
@@ -916,11 +916,11 @@ jobs:
         key: tpcds-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           tpcds-coursier-
-    - name: Install Java 8
+    - name: Install Java 17
       uses: actions/setup-java@v3
       with:
         distribution: temurin
-        java-version: 8
+        java-version: 17
     - name: Cache TPC-DS generated data
       id: cache-tpcds-sf-1
       uses: actions/cache@v3
@@ -1022,11 +1022,11 @@ jobs:
         key: docker-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           docker-integration-coursier-
-    - name: Install Java 8
+    - name: Install Java 17
       uses: actions/setup-java@v3
       with:
         distribution: temurin
-        java-version: 8
+        java-version: 17
     - name: Run tests
       run: |
         ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest
diff --git a/.github/workflows/build_ansi.yml b/.github/workflows/build_ansi.yml
index e67a9262fcd70..57dcbe299332d 100644
--- a/.github/workflows/build_ansi.yml
+++ b/.github/workflows/build_ansi.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / ANSI (master, Hadoop 3, JDK 8, Scala 2.12)"
+name: "Build / ANSI (master, Hadoop 3, JDK 17, Scala 2.12)"
 
 on:
   schedule:
@@ -31,7 +31,7 @@ jobs:
     uses: ./.github/workflows/build_and_test.yml
     if: github.repository == 'apache/spark'
     with:
-      java: 8
+      java: 17
       branch: master
       hadoop: hadoop3
       envs: >-
diff --git a/.github/workflows/build_coverage.yml b/.github/workflows/build_coverage.yml
index aa210f0031866..b534aaef32555 100644
--- a/.github/workflows/build_coverage.yml
+++ b/.github/workflows/build_coverage.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Coverage (master, Scala 2.12, Hadoop 3, JDK 8)"
+name: "Build / Coverage (master, Scala 2.12, Hadoop 17, JDK 8)"
 
 on:
   schedule:
@@ -31,7 +31,7 @@ jobs:
     uses: ./.github/workflows/build_and_test.yml
     if: github.repository == 'apache/spark'
     with:
-      java: 8
+      java: 17
       branch: master
       hadoop: hadoop3
       envs: >-
diff --git a/.github/workflows/build_java11.yml b/.github/workflows/build_java11.yml
deleted file mode 100644
index bf7b2edb45ff3..0000000000000
--- a/.github/workflows/build_java11.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: "Build (master, Scala 2.12, Hadoop 3, JDK 11)"
-
-on:
-  schedule:
-    - cron: '0 16 * * *'
-
-jobs:
-  run-build:
-    permissions:
-      packages: write
-    name: Run
-    uses: ./.github/workflows/build_and_test.yml
-    if: github.repository == 'apache/spark'
-    with:
-      java: 11
-      branch: master
-      hadoop: hadoop3
-      envs: >-
-        {
-          "SKIP_MIMA": "true",
-          "SKIP_UNIDOC": "true"
-        }
-      jobs: >-
-        {
-          "build": "true",
-          "pyspark": "true",
-          "sparkr": "true",
-          "tpcds-1g": "true",
-          "docker-integration-tests": "true"
-        }
diff --git a/.github/workflows/build_rockdb_as_ui_backend.yml b/.github/workflows/build_rockdb_as_ui_backend.yml
index 04e0e7c2e1073..bd3fc95d9f081 100644
--- a/.github/workflows/build_rockdb_as_ui_backend.yml
+++ b/.github/workflows/build_rockdb_as_ui_backend.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / RocksDB as UI Backend (master, Hadoop 3, JDK 8, Scala 2.12)"
+name: "Build / RocksDB as UI Backend (master, Hadoop 3, JDK 17, Scala 2.12)"
 
 on:
   schedule:
@@ -31,7 +31,7 @@ jobs:
     uses: ./.github/workflows/build_and_test.yml
     if: github.repository == 'apache/spark'
     with:
-      java: 8
+      java: 17
       branch: master
       hadoop: hadoop3
       envs: >-
diff --git a/.github/workflows/build_scala213.yml b/.github/workflows/build_scala213.yml
index cae0981ee1e8a..c4bc6339dab94 100644
--- a/.github/workflows/build_scala213.yml
+++ b/.github/workflows/build_scala213.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build (master, Scala 2.13, Hadoop 3, JDK 8)"
+name: "Build (master, Scala 2.13, Hadoop 3, JDK 17)"
 
 on:
   schedule:
@@ -31,7 +31,7 @@ jobs:
     uses: ./.github/workflows/build_and_test.yml
     if: github.repository == 'apache/spark'
     with:
-      java: 8
+      java: 17
       branch: master
       hadoop: hadoop3
       envs: >-
diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
index 9bdb52e4e92da..458c69889fcff 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -32,9 +32,6 @@ jobs:
       matrix:
         branch:
           - master
-          - branch-3.5
-          - branch-3.4
-          - branch-3.3
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -47,11 +44,11 @@ jobs:
         key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           snapshot-maven-
-    - name: Install Java 8
+    - name: Install Java 17
       uses: actions/setup-java@v3
       with:
         distribution: temurin
-        java-version: 8
+        java-version: 17
     - name: Publish snapshot
       env:
         ASF_USERNAME: ${{ secrets.NEXUS_USER }}
diff --git a/docs/building-spark.md b/docs/building-spark.md
index bbbc51d8c22c9..8f026adbe2037 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -27,7 +27,7 @@ license: |
 ## Apache Maven
 
 The Maven-based build is the build of reference for Apache Spark.
-Building Spark using Maven requires Maven 3.9.4 and Java 8/11/17.
+Building Spark using Maven requires Maven 3.9.4 and Java 17.
 Spark requires Scala 2.12/2.13; support for Scala 2.11 was removed in Spark 3.0.0.
 
 ### Setting up Maven's Memory Usage
diff --git a/docs/index.md b/docs/index.md
index bd77fd75a0b3b..d65ae1f7abe50 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -34,12 +34,11 @@ source, visit [Building Spark](building-spark.html).
 
 Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
 
-Spark runs on Java 8/11/17, Scala 2.12/2.13, Python 3.8+, and R 3.5+.
-Java 8 prior to version 8u371 support is deprecated as of Spark 3.5.0.
+Spark runs on Java 17, Scala 2.12/2.13, Python 3.8+, and R 3.5+.
 When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for.
 For example, when using Scala 2.13, use Spark compiled for 2.13, and compile code/applications for Scala 2.13 as well.
 
-For Java 11, setting `-Dio.netty.tryReflectionSetAccessible=true` is required for the Apache Arrow library. This prevents the `java.lang.UnsupportedOperationException: sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) not available` error when Apache Arrow uses Netty internally.
+For Java 17, setting `-Dio.netty.tryReflectionSetAccessible=true` is required for the Apache Arrow library. This prevents the `java.lang.UnsupportedOperationException: sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) not available` error when Apache Arrow uses Netty internally.
 
 # Running the Examples and Shell
 
diff --git a/pom.xml b/pom.xml
index 971cb07ea40ea..bff3e2a3c0da9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,7 +112,7 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <java.version>1.8</java.version>
+    <java.version>17</java.version>
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <maven.version>3.9.4</maven.version>
@@ -2933,7 +2933,7 @@
               <arg>-deprecation</arg>
               <arg>-feature</arg>
               <arg>-explaintypes</arg>
-              <arg>-target:jvm-1.8</arg>
+              <arg>-target:17</arg>
               <arg>-Xfatal-warnings</arg>
               <arg>-Ywarn-unused:imports</arg>
               <arg>-P:silencer:globalFilters=.*deprecated.*</arg>
@@ -3645,7 +3645,7 @@
                   <arg>-deprecation</arg>
                   <arg>-feature</arg>
                   <arg>-explaintypes</arg>
-                  <arg>-target:jvm-1.8</arg>
+                  <arg>-target:17</arg>
                   <arg>-Wconf:cat=deprecation:wv,any:e</arg>
                   <arg>-Wunused:imports</arg>
                   <!--

From 4c11396e7876683e822f4ffd798179c52103f212 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 11:28:36 +0800
Subject: [PATCH 02/14] add old branch publish_snapshot

---
 .github/workflows/publish_snapshot_java8.yml | 61 ++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .github/workflows/publish_snapshot_java8.yml

diff --git a/.github/workflows/publish_snapshot_java8.yml b/.github/workflows/publish_snapshot_java8.yml
new file mode 100644
index 0000000000000..42c6e45097a88
--- /dev/null
+++ b/.github/workflows/publish_snapshot_java8.yml
@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Publish Snapshot using Java 8
+
+on:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  publish-snapshot:
+    if: github.repository == 'apache/spark'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - branch-3.5
+          - branch-3.4
+          - branch-3.3
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ matrix.branch }}
+    - name: Cache Maven local repository
+      uses: actions/cache@v3
+      with:
+        path: ~/.m2/repository
+        key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          snapshot-maven-
+    - name: Install Java 8
+      uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 8
+    - name: Publish snapshot
+      env:
+        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+        GPG_KEY: "not_used"
+        GPG_PASSPHRASE: "not_used"
+        GIT_REF: ${{ matrix.branch }}
+      run: ./dev/create-release/release-build.sh publish-snapshot

From f7ae713ae970f4f017d1fa11b6655dccbc313aee Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 11:35:40 +0800
Subject: [PATCH 03/14] remove 11 maven build

---
 .github/workflows/build_and_test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 7104a29108940..9b69a2032133c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -778,7 +778,6 @@ jobs:
       fail-fast: false
       matrix:
         java:
-          - 11
           - 17
           - 21-ea
     runs-on: ubuntu-22.04

From cb5f8de9e07e2cf1796d980f602adce1f0a9367c Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 13:20:37 +0800
Subject: [PATCH 04/14] test with Scala 2.13

---
 .github/workflows/build_and_test.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 9b69a2032133c..29d2729c20d2f 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -207,6 +207,7 @@ jobs:
       GITHUB_PREV_SHA: ${{ github.event.before }}
       SPARK_LOCAL_IP: localhost
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -385,6 +386,7 @@ jobs:
       SKIP_PACKAGING: true
       METASPACE_SIZE: 1g
       BRANCH: ${{ inputs.branch }}
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -494,6 +496,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
       SKIP_MIMA: true
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -989,6 +992,7 @@ jobs:
       ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:21.3.0
       SKIP_MIMA: true
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -1109,7 +1113,8 @@ jobs:
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
           kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
-          build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
+          dev/change-scala-version.sh 2.13
+          build/sbt -Pscala-2.13 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files
         if: ${{ !success() }}
         uses: actions/upload-artifact@v3

From 72b51ada98796d149a4cf5e62d9367926210f1a4 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 13:24:29 +0800
Subject: [PATCH 05/14] Revert "test with Scala 2.13"

This reverts commit cb5f8de9e07e2cf1796d980f602adce1f0a9367c.
---
 .github/workflows/build_and_test.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 29d2729c20d2f..9b69a2032133c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -207,7 +207,6 @@ jobs:
       GITHUB_PREV_SHA: ${{ github.event.before }}
       SPARK_LOCAL_IP: localhost
       SKIP_PACKAGING: true
-      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -386,7 +385,6 @@ jobs:
       SKIP_PACKAGING: true
       METASPACE_SIZE: 1g
       BRANCH: ${{ inputs.branch }}
-      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -496,7 +494,6 @@ jobs:
       SPARK_LOCAL_IP: localhost
       SKIP_MIMA: true
       SKIP_PACKAGING: true
-      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -992,7 +989,6 @@ jobs:
       ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:21.3.0
       SKIP_MIMA: true
       SKIP_PACKAGING: true
-      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -1113,8 +1109,7 @@ jobs:
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
           kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
-          dev/change-scala-version.sh 2.13
-          build/sbt -Pscala-2.13 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
+          build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files
         if: ${{ !success() }}
         uses: actions/upload-artifact@v3

From 4c6f254327237b9075724df7ff5edffeec8c9ce5 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 13:25:00 +0800
Subject: [PATCH 06/14] SparkBuild

---
 project/SparkBuild.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index dc2e20c644b2a..4c41d3a7de75a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -366,7 +366,7 @@ object SparkBuild extends PomBuild {
     ),
 
     (Compile / scalacOptions) ++= Seq(
-      s"-target:jvm-${javaVersion.value}",
+      s"-target:${javaVersion.value}",
       "-sourcepath", (ThisBuild / baseDirectory).value.getAbsolutePath  // Required for relative source links in scaladoc
     ),
 

From 50d7693c5dce4c2ec7d0505f4e0b7178faa45ddd Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 13:27:40 +0800
Subject: [PATCH 07/14] Revert "Revert "test with Scala 2.13""

This reverts commit 72b51ada98796d149a4cf5e62d9367926210f1a4.
---
 .github/workflows/build_and_test.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 9b69a2032133c..29d2729c20d2f 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -207,6 +207,7 @@ jobs:
       GITHUB_PREV_SHA: ${{ github.event.before }}
       SPARK_LOCAL_IP: localhost
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -385,6 +386,7 @@ jobs:
       SKIP_PACKAGING: true
       METASPACE_SIZE: 1g
       BRANCH: ${{ inputs.branch }}
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -494,6 +496,7 @@ jobs:
       SPARK_LOCAL_IP: localhost
       SKIP_MIMA: true
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -989,6 +992,7 @@ jobs:
       ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:21.3.0
       SKIP_MIMA: true
       SKIP_PACKAGING: true
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -1109,7 +1113,8 @@ jobs:
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
           kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
-          build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
+          dev/change-scala-version.sh 2.13
+          build/sbt -Pscala-2.13 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files
         if: ${{ !success() }}
         uses: actions/upload-artifact@v3

From 7ce64d83d231b3ad2af6e0ba34816eb2759f2b82 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 16:00:04 +0800
Subject: [PATCH 08/14] tpcds test scala 2.13

---
 .github/workflows/build_and_test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 29d2729c20d2f..c71fc9fb84def 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -887,6 +887,7 @@ jobs:
     timeout-minutes: 300
     env:
       SPARK_LOCAL_IP: localhost
+      SCALA_PROFILE: scala2.13
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3

From bbeba9a49a0f30c1f161b0bdd74771608a9f6419 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 20 Sep 2023 16:34:57 +0800
Subject: [PATCH 09/14] fix comments

---
 .github/workflows/build_coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_coverage.yml b/.github/workflows/build_coverage.yml
index b534aaef32555..8388b7fdd9876 100644
--- a/.github/workflows/build_coverage.yml
+++ b/.github/workflows/build_coverage.yml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: "Build / Coverage (master, Scala 2.12, Hadoop 17, JDK 8)"
+name: "Build / Coverage (master, Scala 2.12, Hadoop 3, JDK 17)"
 
 on:
   schedule:

From 278cac66ca017e4bc1dca0de4e9d71e7049d4844 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Thu, 21 Sep 2023 20:43:47 +0800
Subject: [PATCH 10/14] merge publish snapshot java 8 into publish snapshot

---
 .github/workflows/publish_snapshot.yml       | 10 ++++
 .github/workflows/publish_snapshot_java8.yml | 61 --------------------
 2 files changed, 10 insertions(+), 61 deletions(-)
 delete mode 100644 .github/workflows/publish_snapshot_java8.yml

diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
index 458c69889fcff..7ed836f016b9b 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -32,6 +32,9 @@ jobs:
       matrix:
         branch:
           - master
+          - branch-3.5
+          - branch-3.4
+          - branch-3.3
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v3
@@ -44,7 +47,14 @@ jobs:
         key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           snapshot-maven-
+    - name: Install Java 8 for branch-3.x
+      if: matrix.branch == 'branch-3.5' || matrix.branch == 'branch-3.4' || matrix.branch == 'branch-3.3'
+      uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 8
     - name: Install Java 17
+      if: matrix.branch != 'branch-3.5' && matrix.branch != 'branch-3.4' && matrix.branch != 'branch-3.3'
       uses: actions/setup-java@v3
       with:
         distribution: temurin
diff --git a/.github/workflows/publish_snapshot_java8.yml b/.github/workflows/publish_snapshot_java8.yml
deleted file mode 100644
index 42c6e45097a88..0000000000000
--- a/.github/workflows/publish_snapshot_java8.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: Publish Snapshot using Java 8
-
-on:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  publish-snapshot:
-    if: github.repository == 'apache/spark'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        branch:
-          - branch-3.5
-          - branch-3.4
-          - branch-3.3
-    steps:
-    - name: Checkout Spark repository
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ matrix.branch }}
-    - name: Cache Maven local repository
-      uses: actions/cache@v3
-      with:
-        path: ~/.m2/repository
-        key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          snapshot-maven-
-    - name: Install Java 8
-      uses: actions/setup-java@v3
-      with:
-        distribution: temurin
-        java-version: 8
-    - name: Publish snapshot
-      env:
-        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
-        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
-        GPG_KEY: "not_used"
-        GPG_PASSPHRASE: "not_used"
-        GIT_REF: ${{ matrix.branch }}
-      run: ./dev/create-release/release-build.sh publish-snapshot

From c0a330d1aaf802ebdf3c030dbd32611bd41f3f29 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 22 Sep 2023 13:14:11 +0800
Subject: [PATCH 11/14] remove -Pscala-2.13

---
 .github/workflows/build_and_test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 211f4213bec47..f01534a3d1700 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1061,8 +1061,7 @@ jobs:
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
           kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
-          dev/change-scala-version.sh 2.13
-          build/sbt -Pscala-2.13 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
+          build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files
         if: ${{ !success() }}
         uses: actions/upload-artifact@v3

From 6ca32e3379df9da0f0142b795cd4d7e6402fa0f8 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 22 Sep 2023 13:30:24 +0800
Subject: [PATCH 12/14] change to use inputs.java

---
 .github/workflows/build_and_test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index f01534a3d1700..72934b4fafa85 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -649,11 +649,11 @@ jobs:
         if [ -f ./dev/free_disk_space_container ]; then
           ./dev/free_disk_space_container
         fi
-    - name: Install Java 17
+    - name: Install Java ${{ inputs.java }}
       uses: actions/setup-java@v3
       with:
         distribution: zulu
-        java-version: 17
+        java-version: ${{ inputs.java }}
     - name: License test
       run: ./dev/check-license
     - name: Dependencies test
@@ -867,11 +867,11 @@ jobs:
         key: tpcds-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           tpcds-coursier-
-    - name: Install Java 17
+    - name: Install Java ${{ inputs.java }}
       uses: actions/setup-java@v3
       with:
         distribution: zulu
-        java-version: 17
+        java-version: ${{ inputs.java }}
     - name: Cache TPC-DS generated data
       id: cache-tpcds-sf-1
       uses: actions/cache@v3
@@ -973,11 +973,11 @@ jobs:
         key: docker-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           docker-integration-coursier-
-    - name: Install Java 17
+    - name: Install Java ${{ inputs.java }}
       uses: actions/setup-java@v3
       with:
         distribution: zulu
-        java-version: 17
+        java-version: ${{ inputs.java }}
     - name: Run tests
       run: |
         ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest

From 2ea31602d8996abf60e45293d31b4fa3fed27658 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 22 Sep 2023 14:45:53 +0800
Subject: [PATCH 13/14] add more doc

---
 docs/security.md                               | 10 +++++-----
 python/docs/source/getting_started/install.rst |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/security.md b/docs/security.md
index 3c6fd507fec6d..30cfd660c41bf 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -498,8 +498,8 @@ replaced with one of the above namespaces.
       A comma-separated list of ciphers. The specified ciphers must be supported by JVM.
 
       <br />The reference list of protocols can be found in the "JSSE Cipher Suite Names" section
-      of the Java security guide. The list for Java 8 can be found at
-      <a href="https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#ciphersuites">this</a>
+      of the Java security guide. The list for Java 17 can be found at
+      <a href="https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html#jsse-cipher-suite-names">this</a>
       page.
 
       <br />Note: If not set, the default cipher suite for the JRE will be used.
@@ -537,8 +537,8 @@ replaced with one of the above namespaces.
       TLS protocol to use. The protocol must be supported by JVM.
 
       <br />The reference list of protocols can be found in the "Additional JSSE Standard Names"
-      section of the Java security guide. For Java 8, the list can be found at
-      <a href="https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames">this</a>
+      section of the Java security guide. For Java 17, the list can be found at
+      <a href="https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html#additional-jsse-standard-names">this</a>
       page.
     </td>
   </tr>
@@ -591,7 +591,7 @@ Or via SparkConf "spark.hadoop.hadoop.security.credential.provider.path=jceks://
 ## Preparing the key stores
 
 Key stores can be generated by `keytool` program. The reference documentation for this tool for
-Java 8 is [here](https://docs.oracle.com/javase/8/docs/technotes/tools/unix/keytool.html).
+Java 17 is [here](https://docs.oracle.com/en/java/javase/17/docs/specs/man/keytool.html).
 The most basic steps to configure the key stores and the trust store for a Spark Standalone
 deployment mode is as follows:
 
diff --git a/python/docs/source/getting_started/install.rst b/python/docs/source/getting_started/install.rst
index 909f9cff3e38d..a747d0cdebd73 100644
--- a/python/docs/source/getting_started/install.rst
+++ b/python/docs/source/getting_started/install.rst
@@ -164,7 +164,7 @@ Package                    Supported version Note
 `googleapis-common-protos` ==1.56.4                  Required for Spark Connect
 ========================== ========================= ======================================================================================
 
-Note that PySpark requires Java 8 or later with ``JAVA_HOME`` properly set.  
-If using JDK 11, set ``-Dio.netty.tryReflectionSetAccessible=true`` for Arrow related features and refer
+Note that PySpark requires Java 17 or later with ``JAVA_HOME`` properly set.
+Need set ``-Dio.netty.tryReflectionSetAccessible=true`` for Arrow related features and refer
 to |downloading|_.
 

From 54f40ab6e7e2c349904bc1f97f1be4555a4f371c Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 22 Sep 2023 17:12:06 +0800
Subject: [PATCH 14/14] infra Dockerfile install openjdk-17-jdk-headless

---
 dev/infra/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile
index d196d0e97c579..767606d299a3e 100644
--- a/dev/infra/Dockerfile
+++ b/dev/infra/Dockerfile
@@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \
     pkg-config  \
     curl  \
     wget  \
-    openjdk-8-jdk  \
+    openjdk-17-jdk-headless  \
     gfortran  \
     libopenblas-dev  \
     liblapack-dev \