From 812f73507e231843705682c1c7229898f769f422 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 27 May 2025 17:29:36 -0600 Subject: [PATCH 1/3] Enable Spark 4 tests on Linux --- .github/actions/java-test/action.yaml | 14 ++++- .github/workflows/pr_build_linux_spark4.yml | 70 ++++++++++++++++----- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 53775da0df..90286870e1 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -18,6 +18,10 @@ name: "Java Test" description: "Run Java tests" inputs: + suites: + description: 'Which test suites to run' + required: false + default: '' maven_opts: description: 'Maven options passed to the mvn command' required: false @@ -51,10 +55,18 @@ runs: run: | ./mvnw -B compile test-compile scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ inputs.maven_opts }} - - name: Run tests + - name: Run all tests shell: bash + if: ${{ inputs.suite == '' }} run: | MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} + - name: Run specified tests + shell: bash + if: ${{ inputs.suite != '' }} + run: | + MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)" + echo "Running with MAVEN_SUITES=$MAVEN_SUITES" + MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }} - name: Upload crash logs if: failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/pr_build_linux_spark4.yml b/.github/workflows/pr_build_linux_spark4.yml index dc0a96c8d8..7a34425c3d 100644 --- a/.github/workflows/pr_build_linux_spark4.yml +++ b/.github/workflows/pr_build_linux_spark4.yml @@ -23,17 +23,16 @@ concurrency: on: -# temporarily disabled on PRs due to https://github.com/apache/datafusion-comet/issues/1786 -# push: -# paths-ignore: -# - "doc/**" -# - "docs/**" -# - "**.md" -# pull_request: -# paths-ignore: -# - "doc/**" -# - "docs/**" -# - "**.md" + push: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + pull_request: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: @@ -50,10 +49,49 @@ jobs: java_version: [17] test-target: [java] spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} + suites: + - name: "fuzz" + value: | + org.apache.comet.CometFuzzTestSuite + org.apache.comet.DataGeneratorSuite + - name: "shuffle" + value: | + org.apache.comet.exec.CometShuffleSuite + org.apache.comet.exec.CometShuffle4_0Suite + org.apache.comet.exec.CometNativeShuffleSuite + org.apache.comet.exec.CometShuffleEncryptionSuite + org.apache.comet.exec.CometShuffleManagerSuite + org.apache.comet.exec.CometAsyncShuffleSuite + org.apache.comet.exec.DisableAQECometShuffleSuite + org.apache.comet.exec.DisableAQECometAsyncShuffleSuite + - name: "parquet" + value: | + org.apache.comet.parquet.ParquetReadV1Suite + org.apache.comet.parquet.ParquetReadV2Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite + org.apache.spark.sql.comet.ParquetEncryptionITCase + org.apache.comet.exec.CometNativeReaderSuite + - name: "exec" + value: | + org.apache.comet.exec.CometAggregateSuite + org.apache.comet.exec.CometExec3_4PlusSuite + org.apache.comet.exec.CometExecSuite + org.apache.comet.exec.CometJoinSuite + org.apache.comet.CometArrayExpressionSuite + org.apache.comet.CometCastSuite + org.apache.comet.CometExpressionSuite + org.apache.comet.CometExpressionCoverageSuite + org.apache.comet.CometNativeSuite + org.apache.comet.CometSparkSessionExtensionsSuite + org.apache.spark.CometPluginsSuite + org.apache.spark.sql.CometTPCDSQuerySuite + org.apache.spark.sql.CometTPCDSQueryTestSuite + org.apache.spark.sql.CometTPCHQuerySuite + org.apache.spark.sql.comet.CometPlanStabilitySuite + org.apache.spark.sql.comet.CometTaskMetricsSuite fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}] runs-on: ${{ matrix.os }} container: image: amd64/rust @@ -67,6 +105,6 @@ jobs: - name: Java test steps uses: ./.github/actions/java-test with: - maven_opts: -Pspark-${{ matrix.spark-version }} + suites: ${{ matrix.suites.value }} + maven_opts: "-Pspark-${{ matrix.spark-version }}" upload-test-reports: true - From d3892a15f2dc1f437bfad60ccd710520093e05eb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 27 May 2025 17:34:50 -0600 Subject: [PATCH 2/3] add more tests --- .github/workflows/pr_build_linux_spark4.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_build_linux_spark4.yml b/.github/workflows/pr_build_linux_spark4.yml index 7a34425c3d..0c143eb53d 100644 --- a/.github/workflows/pr_build_linux_spark4.yml +++ b/.github/workflows/pr_build_linux_spark4.yml @@ -85,10 +85,14 @@ jobs: org.apache.comet.CometNativeSuite org.apache.comet.CometSparkSessionExtensionsSuite org.apache.spark.CometPluginsSuite + org.apache.spark.CometPluginsDefaultSuite + org.apache.spark.CometPluginsNonOverrideSuite + org.apache.spark.CometPluginsUnifiedModeOverrideSuite org.apache.spark.sql.CometTPCDSQuerySuite org.apache.spark.sql.CometTPCDSQueryTestSuite org.apache.spark.sql.CometTPCHQuerySuite - org.apache.spark.sql.comet.CometPlanStabilitySuite + org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite + org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite org.apache.spark.sql.comet.CometTaskMetricsSuite fail-fast: false name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}] From 8c95fb3b066a324a7c2f1026498442ad95a99b79 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 27 May 2025 21:52:03 -0600 Subject: [PATCH 3/3] fix --- .github/actions/java-test/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 90286870e1..c0ff974aa3 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -57,16 +57,16 @@ runs: - name: Run all tests shell: bash - if: ${{ inputs.suite == '' }} + if: ${{ inputs.suites == '' }} run: | MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} - name: Run specified tests shell: bash - if: ${{ inputs.suite != '' }} + if: ${{ inputs.suites != '' }} run: | MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)" echo "Running with MAVEN_SUITES=$MAVEN_SUITES" - MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }} + MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} - name: Upload crash logs if: failure() uses: actions/upload-artifact@v4