diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index c0ff974aa3..634c42a640 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -26,6 +26,10 @@ inputs: description: 'Maven options passed to the mvn command' required: false default: '' + scan_impl: + description: 'The default Parquet scan implementation' + required: false + default: 'native_comet' upload-test-reports: description: 'Whether to upload test results including coverage to GitHub' required: false @@ -58,11 +62,15 @@ runs: - name: Run all tests shell: bash if: ${{ inputs.suites == '' }} + env: + COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }} run: | MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} - name: Run specified tests shell: bash if: ${{ inputs.suites != '' }} + env: + COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }} run: | MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)" echo "Running with MAVEN_SUITES=$MAVEN_SUITES" diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml deleted file mode 100644 index 308dd2d68c..0000000000 --- a/.github/workflows/pr_build.yml +++ /dev/null @@ -1,317 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PR Build - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -env: - RUST_VERSION: stable - -jobs: - - # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release - linux-test: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - # run Comet tests for older 3.5.x releases with one java and scala versions - linux-test-35-minor-versions: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [17] - test-target: [java] - spark-version: ['3.5'] - spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - linux-test-native-datafusion-scan: - strategy: - matrix: - os: [ubuntu-latest] - scan_impl: ['native_datafusion', 'native_iceberg_compat'] - java_version: [17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} - runs-on: ${{ matrix.os }} - env: - COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - linux-test-with-old-spark: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-test: - strategy: - matrix: - os: [macos-13] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-aarch64-test: - strategy: - matrix: - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-test-with-spark4_0: - strategy: - matrix: - os: [macos-13] - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - macos-aarch64-test-with-spark4_0: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - macos-aarch64-test-with-old-spark: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - exclude: - - java_version: 8 - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml new file mode 100644 index 0000000000..732b5b4d15 --- /dev/null +++ b/.github/workflows/pr_build_linux.yml @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PR Build (Linux) + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + pull_request: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + # manual trigger + # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: + +env: + RUST_VERSION: stable + +jobs: + + # Run Rust tests once per JDK version + linux-test-rust: + strategy: + matrix: + os: [ubuntu-latest] + java_version: [11, 17] + fail-fast: false + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-rust + runs-on: ${{ matrix.os }} + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.java_version }} + - name: Rust test steps + uses: ./.github/actions/rust-test + + linux-test: + strategy: + matrix: + os: [ubuntu-latest] + # the goal with these profiles is to get coverage of all Java, Scala, and Spark + # versions without testing all possible combinations, which would be overkill + profile: + - name: "Spark 3.4, JDK 11, Scala 2.12" + java_version: "11" + maven_opts: "-Pspark-3.4 -Pscala-2.12" + scan_impl: "native_comet" + + - name: "Spark 3.5, JDK 11, Scala 2.12" + java_version: "17" + maven_opts: "-Pspark-3.5 -Pscala-2.12" + scan_impl: "native_comet" + + - name: "Spark 3.5, JDK 17, Scala 2.13" + java_version: "17" + maven_opts: "-Pspark-3.5 -Pscala-2.13" + scan_impl: "native_comet" + + - name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion" + java_version: "17" + maven_opts: "-Pspark-3.5 -Pscala-2.12" + scan_impl: "native_datafusion" + + - name: "Spark 3.5, JDK 17, Scala 2.12 native_iceberg_compat" + java_version: "17" + maven_opts: "-Pspark-3.5 -Pscala-2.12" + scan_impl: "native_iceberg_compat" + + - name: "Spark 4.0, JDK 17" + java_version: "17" + maven_opts: "-Pspark-4.0" + scan_impl: "native_comet" + suite: + - name: "fuzz" + value: | + org.apache.comet.CometFuzzTestSuite + org.apache.comet.DataGeneratorSuite + - name: "shuffle" + value: | + org.apache.comet.exec.CometShuffleSuite + org.apache.comet.exec.CometShuffle4_0Suite + org.apache.comet.exec.CometNativeShuffleSuite + org.apache.comet.exec.CometShuffleEncryptionSuite + org.apache.comet.exec.CometShuffleManagerSuite + org.apache.comet.exec.CometAsyncShuffleSuite + org.apache.comet.exec.DisableAQECometShuffleSuite + org.apache.comet.exec.DisableAQECometAsyncShuffleSuite + - name: "parquet" + value: | + org.apache.comet.parquet.ParquetReadV1Suite + org.apache.comet.parquet.ParquetReadV2Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite + org.apache.spark.sql.comet.ParquetEncryptionITCase + org.apache.comet.exec.CometNativeReaderSuite + - name: "exec" + value: | + org.apache.comet.exec.CometAggregateSuite + org.apache.comet.exec.CometExec3_4PlusSuite + org.apache.comet.exec.CometExecSuite + org.apache.comet.exec.CometJoinSuite + org.apache.comet.CometArrayExpressionSuite + org.apache.comet.CometCastSuite + org.apache.comet.CometExpressionSuite + org.apache.comet.CometExpressionCoverageSuite + org.apache.comet.CometNativeSuite + org.apache.comet.CometSparkSessionExtensionsSuite + org.apache.comet.CometStringExpressionSuite + org.apache.spark.CometPluginsSuite + org.apache.spark.CometPluginsDefaultSuite + org.apache.spark.CometPluginsNonOverrideSuite + org.apache.spark.CometPluginsUnifiedModeOverrideSuite + org.apache.spark.sql.CometTPCDSQuerySuite + org.apache.spark.sql.CometTPCDSQueryTestSuite + org.apache.spark.sql.CometTPCHQuerySuite + org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite + org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite + org.apache.spark.sql.comet.CometTaskMetricsSuite + fail-fast: false + name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name }}] + runs-on: ${{ matrix.os }} + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.profile.java_version }} + - name: Java test steps + uses: ./.github/actions/java-test + with: + suites: ${{ matrix.suite.value }} + maven_opts: ${{ matrix.profile.maven_opts }} + scan_impl: ${{ matrix.profile.scan_impl }} + upload-test-reports: true \ No newline at end of file diff --git a/.github/workflows/pr_build_linux_spark4.yml b/.github/workflows/pr_build_macos.yml similarity index 86% rename from .github/workflows/pr_build_linux_spark4.yml rename to .github/workflows/pr_build_macos.yml index 0c143eb53d..85b40ac5b5 100644 --- a/.github/workflows/pr_build_linux_spark4.yml +++ b/.github/workflows/pr_build_macos.yml @@ -15,14 +15,13 @@ # specific language governing permissions and limitations # under the License. -name: PR Build (Linux + Spark 4) +name: PR Build (macOS) concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: - push: paths-ignore: - "doc/**" @@ -42,14 +41,13 @@ env: jobs: - linux-test-with-spark4_0: + macos-aarch64-test: strategy: matrix: - os: [ubuntu-latest] java_version: [17] - test-target: [java] - spark-version: ['4.0'] - suites: + spark-version: ['3.4', '3.5', '4.0'] + scala-version: ['2.13'] + suite: - name: "fuzz" value: | org.apache.comet.CometFuzzTestSuite @@ -84,6 +82,7 @@ jobs: org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometNativeSuite org.apache.comet.CometSparkSessionExtensionsSuite + org.apache.comet.CometStringExpressionSuite org.apache.spark.CometPluginsSuite org.apache.spark.CometPluginsDefaultSuite org.apache.spark.CometPluginsNonOverrideSuite @@ -95,20 +94,19 @@ jobs: org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite org.apache.spark.sql.comet.CometTaskMetricsSuite fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}] - runs-on: ${{ matrix.os }} - container: - image: amd64/rust + name: macos-14(Silicon)/ ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}} [${{matrix.suite.name}}] + runs-on: macos-14 steps: - uses: actions/checkout@v4 - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder + uses: ./.github/actions/setup-macos-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java_version }} + jdk-architecture: aarch64 + protoc-architecture: aarch_64 - name: Java test steps uses: ./.github/actions/java-test with: - suites: ${{ matrix.suites.value }} - maven_opts: "-Pspark-${{ matrix.spark-version }}" - upload-test-reports: true + suites: ${{ matrix.suite.value }} + maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} \ No newline at end of file