From f62f8ddae34bc55665b829fad87972098bdefe14 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 12:07:44 -0600 Subject: [PATCH 01/10] Run Comet tests for more Spark versions --- .github/workflows/pr_build.yml | 38 +++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 255fc47c28..87c9fcbd61 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -47,6 +47,7 @@ jobs: java_version: [8, 11, 17] test-target: [rust, java] spark-version: ['3.5'] + spark-patch-version: ['3.5.2', '3.5.3', '3.5.4', '3.5.5'] scala-version: ['2.12', '2.13'] is_push_event: - ${{ github.event_name == 'push' }} @@ -54,7 +55,7 @@ jobs: - java_version: 11 is_push_event: false fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -76,6 +77,41 @@ jobs: # upload test reports only for java 17 upload-test-reports: ${{ matrix.java_version == '17' }} + # run Comet tests for older 3.5.x releases with one java and scala versions + linux-test-35-minor-versions: + strategy: + matrix: + os: [ubuntu-latest] + java_version: [17] + test-target: [java] + spark-version: ['3.5'] + spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] + scala-version: ['2.12'] + is_push_event: + - ${{ github.event_name == 'push' }} + fail-fast: false + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} + runs-on: ${{ matrix.os }} + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.java_version }} + - if: matrix.test-target == 'rust' + name: Rust test steps + uses: ./.github/actions/rust-test + - if: matrix.test-target == 'java' + name: Java test steps + uses: ./.github/actions/java-test + with: + maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" + # upload test reports only for java 17 + upload-test-reports: ${{ matrix.java_version == '17' }} + linux-test-native-datafusion-scan: strategy: matrix: From 6a91eecd729a3ca8f4fb5630a309aeec63f36b2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 12:10:24 -0600 Subject: [PATCH 02/10] update --- .github/workflows/pr_build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 87c9fcbd61..52fda63aee 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -40,6 +40,8 @@ env: RUST_VERSION: stable jobs: + + # run Comet test with full matrix of Jva and Scala versions for the latest supported 3.5.x release linux-test: strategy: matrix: @@ -47,7 +49,6 @@ jobs: java_version: [8, 11, 17] test-target: [rust, java] spark-version: ['3.5'] - spark-patch-version: ['3.5.2', '3.5.3', '3.5.4', '3.5.5'] scala-version: ['2.12', '2.13'] is_push_event: - ${{ github.event_name == 'push' }} @@ -55,7 +56,7 @@ jobs: - java_version: 11 is_push_event: false fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} runs-on: ${{ matrix.os }} container: image: amd64/rust From 3efeca91ee36038224fde43ed2ea551386b07eb4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 12:10:53 -0600 Subject: [PATCH 03/10] update --- .github/workflows/pr_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 52fda63aee..72dd0226cc 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -41,7 +41,7 @@ env: jobs: - # run Comet test with full matrix of Jva and Scala versions for the latest supported 3.5.x release + # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release linux-test: strategy: matrix: From a7e6820ca73dc6e306fce70ef44a7bd6357c8407 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 15:42:11 -0600 Subject: [PATCH 04/10] update docs --- docs/source/user-guide/installation.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index 88b6511316..b250eda3d4 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -30,22 +30,34 @@ Make sure the following requirements are met and software installed on your mach ### Supported Spark Versions -Comet currently supports the following versions of Apache Spark: +Comet currently supports the following versions of Apache Spark. Note that this documentation is for the main +branch. Please refer to the version of this documentation in the appropriate GitHub release branch for released +Comet versions. -- 3.4.x (Java 8/11/17, Scala 2.12/2.13) -- 3.5.x (Java 8/11/17, Scala 2.12/2.13) +We recommend only using Comet with Spark versions where we currently have both Comet and Spark tests enabled in CI. +Other versions may work well enough for development and evaluation purposes. + +| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | +| ------------- | ------------ | ------------- | ----------------- | ----------------- | +| 3.4.3 | 8/11/17 | 2.12/2.13 | Yes | Yes | +| 3.5.2 | 8/11/17 | 2.12/2.13 | Yes | No | +| 3.5.3 | 8/11/17 | 2.12/2.13 | Yes | No | +| 3.5.4 | 8/11/17 | 2.12/2.13 | Yes | Yes | +| 3.5.5 | 8/11/17 | 2.12/2.13 | Yes | Yes | Experimental support is provided for the following versions of Apache Spark and is intended for development/testing use only and should not be used in production yet. -- 4.0.0-preview1 (Java 17/21, Scala 2.13) +| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | +| -------------- | ------------ | ------------- | ----------------- | ----------------- | +| 4.0.0-preview1 | 11/17 | 2.13 | Yes | Yes | Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by Cloud Service Providers. ## Using a Published JAR File -Comet jar files are available in [Maven Central](https://central.sonatype.com/namespace/org.apache.datafusion) for amd64 and arm64 architectures for Linux. For Apple OSX, it +Comet jar files are available in [Maven Central](https://central.sonatype.com/namespace/org.apache.datafusion) for amd64 and arm64 architectures for Linux. For Apple OSX, it is currently necessary to build from source. Here are the direct links for downloading the Comet 0.7.0 jar file. From 423a345d7579aaf0e8036cd7c428178cfb625c3e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 15:45:10 -0600 Subject: [PATCH 05/10] address feedback --- docs/source/user-guide/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index b250eda3d4..a7681137a4 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -50,7 +50,7 @@ use only and should not be used in production yet. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | | -------------- | ------------ | ------------- | ----------------- | ----------------- | -| 4.0.0-preview1 | 11/17 | 2.13 | Yes | Yes | +| 4.0.0-preview1 | 17 | 2.13 | Yes | Yes | Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by Cloud Service Providers. From 5fdbf7f204aa42f95aefcc427c05fa943fb37f15 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 15:45:24 -0600 Subject: [PATCH 06/10] address feedback --- docs/source/user-guide/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index a7681137a4..eee54aaace 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -50,7 +50,7 @@ use only and should not be used in production yet. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | | -------------- | ------------ | ------------- | ----------------- | ----------------- | -| 4.0.0-preview1 | 17 | 2.13 | Yes | Yes | +| 4.0.0-preview1 | 17 | 2.13 | Yes | Yes | Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by Cloud Service Providers. From 8fa43ce626df9f07de0d156fc09ea1f072432275 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 16:15:38 -0600 Subject: [PATCH 07/10] update --- docs/source/user-guide/installation.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index eee54aaace..0822e4d2d1 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -38,13 +38,15 @@ We recommend only using Comet with Spark versions where we currently have both C Other versions may work well enough for development and evaluation purposes. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | -| ------------- | ------------ | ------------- | ----------------- | ----------------- | +| ------------- | ------------ | ------------- |-------------------| ----------------- | | 3.4.3 | 8/11/17 | 2.12/2.13 | Yes | Yes | -| 3.5.2 | 8/11/17 | 2.12/2.13 | Yes | No | -| 3.5.3 | 8/11/17 | 2.12/2.13 | Yes | No | +| 3.5.2 | 8/11/17 | 2.12/2.13 | Partial* | No | +| 3.5.3 | 8/11/17 | 2.12/2.13 | Partial* | No | | 3.5.4 | 8/11/17 | 2.12/2.13 | Yes | Yes | | 3.5.5 | 8/11/17 | 2.12/2.13 | Yes | Yes | +* For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. + Experimental support is provided for the following versions of Apache Spark and is intended for development/testing use only and should not be used in production yet. From 87c608f7f951951ef6ab14f6703809a8b7dfb36c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 16:17:03 -0600 Subject: [PATCH 08/10] update --- docs/source/user-guide/installation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index 0822e4d2d1..e249f928bb 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -38,14 +38,14 @@ We recommend only using Comet with Spark versions where we currently have both C Other versions may work well enough for development and evaluation purposes. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | -| ------------- | ------------ | ------------- |-------------------| ----------------- | +| ------------- | ------------ | ------------- | ----------------- | ----------------- | | 3.4.3 | 8/11/17 | 2.12/2.13 | Yes | Yes | -| 3.5.2 | 8/11/17 | 2.12/2.13 | Partial* | No | -| 3.5.3 | 8/11/17 | 2.12/2.13 | Partial* | No | +| 3.5.2 | 8/11/17 | 2.12/2.13 | Partial\* | No | +| 3.5.3 | 8/11/17 | 2.12/2.13 | Partial\* | No | | 3.5.4 | 8/11/17 | 2.12/2.13 | Yes | Yes | | 3.5.5 | 8/11/17 | 2.12/2.13 | Yes | Yes | -* For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. +- For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. Experimental support is provided for the following versions of Apache Spark and is intended for development/testing use only and should not be used in production yet. From ffebd9dff716f8076e6312a2e89842d63c163b40 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 16:17:33 -0600 Subject: [PATCH 09/10] update --- docs/source/user-guide/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index e249f928bb..166fce082c 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -38,11 +38,11 @@ We recommend only using Comet with Spark versions where we currently have both C Other versions may work well enough for development and evaluation purposes. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI | -| ------------- | ------------ | ------------- | ----------------- | ----------------- | +| ------------- | ------------ | ------------- |-------------------| ----------------- | | 3.4.3 | 8/11/17 | 2.12/2.13 | Yes | Yes | | 3.5.2 | 8/11/17 | 2.12/2.13 | Partial\* | No | | 3.5.3 | 8/11/17 | 2.12/2.13 | Partial\* | No | -| 3.5.4 | 8/11/17 | 2.12/2.13 | Yes | Yes | +| 3.5.4 | 8/11/17 | 2.12/2.13 | Partial\* | Yes | | 3.5.5 | 8/11/17 | 2.12/2.13 | Yes | Yes | - For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. From bc9170a78018d40089132dca4ae96fdc55a9a34d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 31 Mar 2025 16:18:00 -0600 Subject: [PATCH 10/10] update --- docs/source/user-guide/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/installation.md b/docs/source/user-guide/installation.md index 166fce082c..75f02b476f 100644 --- a/docs/source/user-guide/installation.md +++ b/docs/source/user-guide/installation.md @@ -45,7 +45,7 @@ Other versions may work well enough for development and evaluation purposes. | 3.5.4 | 8/11/17 | 2.12/2.13 | Partial\* | Yes | | 3.5.5 | 8/11/17 | 2.12/2.13 | Yes | Yes | -- For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. +\* For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI. Experimental support is provided for the following versions of Apache Spark and is intended for development/testing use only and should not be used in production yet.