diff --git a/.github/workflows/e2e-tests-flink-1.x-jdk11.yml b/.github/workflows/e2e-tests-flink-1.x-jdk11.yml index 8c0c0ce42079..d0ccdf45ab2d 100644 --- a/.github/workflows/e2e-tests-flink-1.x-jdk11.yml +++ b/.github/workflows/e2e-tests-flink-1.x-jdk11.yml @@ -52,7 +52,7 @@ jobs: distribution: 'temurin' - name: Build Flink - run: mvn -T 1C -B clean install -DskipTests -Pflink1 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }},java11 + run: mvn -T 1C -B clean install -DskipTests -Pflink1,spark3 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }},java11 - name: Test Flink run: | @@ -62,9 +62,9 @@ jobs: echo "JVM timezone is set to $jvm_timezone" profile="flink-${{ matrix.flink_version }}" if [ "${{ matrix.flink_version }}" = "${{ matrix.flink_version[-1] }}" ]; then - mvn -T 1C -B test -Pflink1 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pjava11 + mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pjava11 else - mvn -T 1C -B test -Pflink1 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile},java11 + mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile},java11 fi env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-flink-1.x.yml b/.github/workflows/e2e-tests-flink-1.x.yml index 1431ebd5ccd2..0b25004b32c5 100644 --- a/.github/workflows/e2e-tests-flink-1.x.yml +++ b/.github/workflows/e2e-tests-flink-1.x.yml @@ -52,7 +52,7 @@ jobs: distribution: 'temurin' - name: Build Flink - run: mvn -T 2C -B clean install -DskipTests -Pflink1 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }} + run: mvn -T 2C -B clean install -DskipTests -Pflink1,spark3 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }} - name: Test Flink run: | @@ -62,9 +62,9 @@ jobs: echo "JVM timezone is set to $jvm_timezone" profile="flink-${{ matrix.flink_version }}" if [ "${{ matrix.flink_version }}" = "${{ matrix.flink_version[-1] }}" ]; then - mvn -T 1C -B test -Pflink1 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone else - mvn -T 1C -B test -Pflink1 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile} + mvn -T 1C -B test -Pflink1,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile} fi env: MAVEN_OPTS: -Xmx4096m diff --git a/.github/workflows/e2e-tests-flink-2.x-jdk11.yml b/.github/workflows/e2e-tests-flink-2.x-jdk11.yml index c1667c482634..175895270054 100644 --- a/.github/workflows/e2e-tests-flink-2.x-jdk11.yml +++ b/.github/workflows/e2e-tests-flink-2.x-jdk11.yml @@ -48,7 +48,9 @@ jobs: distribution: 'temurin' - name: Build Flink - run: mvn -T 1C -B clean install -DskipTests -Pflink2 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }},java11 + run: | + mvn -T 1C -B clean install -DskipTests -Pflink1,spark3 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }},java11 + mvn -T 1C -B clean install -DskipTests -Pflink2,spark3 -pl paimon-e2e-tests -am -Pflink-${{ matrix.flink_version }},java11 - name: Test Flink run: | @@ -58,9 +60,9 @@ jobs: echo "JVM timezone is set to $jvm_timezone" profile="flink-${{ matrix.flink_version }}" if [ "${{ matrix.flink_version }}" = "${{ matrix.flink_version[-1] }}" ]; then - mvn -T 1C -B test -Pflink2 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pjava11 + mvn -T 1C -B test -Pflink2,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pjava11 else - mvn -T 1C -B test -Pflink2 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile},java11 + mvn -T 1C -B test -Pflink2,spark3 -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -P${profile},java11 fi env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 8bad09241328..1dbd533f7eaf 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -64,6 +64,6 @@ jobs: echo "$ASF_PASSWORD" >> $tmp_settings echo "" >> $tmp_settings - mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark3 + mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark3,flink1 rm $tmp_settings diff --git a/.github/workflows/utitcase-flink-1.x-jdk11.yml b/.github/workflows/utitcase-flink-1.x-jdk11.yml index 66aafb2ade4c..faf3c88cbd65 100644 --- a/.github/workflows/utitcase-flink-1.x-jdk11.yml +++ b/.github/workflows/utitcase-flink-1.x-jdk11.yml @@ -45,7 +45,7 @@ jobs: java-version: ${{ env.JDK_VERSION }} distribution: 'temurin' - name: Build Flink - run: mvn -T 1C -B clean install -DskipTests -Pflink1 + run: mvn -T 1C -B clean install -DskipTests -Pflink1,spark3 - name: Test Flink run: | # run tests with random timezone to find out timezone related bugs @@ -57,6 +57,6 @@ jobs: test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" - mvn -T 1C -B clean install -Pflink1 -pl "${test_modules}" -Duser.timezone=$jvm_timezone + mvn -T 1C -B clean install -Pflink1,spark3 -pl "${test_modules}" -Duser.timezone=$jvm_timezone env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/utitcase-flink-1.x.yml b/.github/workflows/utitcase-flink-1.x.yml index fe249a45d330..87e8380dfba1 100644 --- a/.github/workflows/utitcase-flink-1.x.yml +++ b/.github/workflows/utitcase-flink-1.x.yml @@ -54,7 +54,7 @@ jobs: run: | COMPILE_MODULE="org.apache.paimon:paimon-flink-${{ matrix.flink_version }}" echo "Start compiling modules: $COMPILE_MODULE" - mvn -T 2C -B clean install -DskipTests -Pflink1 -pl "${COMPILE_MODULE}" -am + mvn -T 2C -B clean install -DskipTests -Pflink1,spark3 -pl "${COMPILE_MODULE}" -am - name: Test Flink run: | @@ -63,7 +63,7 @@ jobs: echo "JVM timezone is set to $jvm_timezone" TEST_MODULE="org.apache.paimon:paimon-flink-${{ matrix.flink_version }}" echo "Start testing module: $TEST_MODULE" - mvn -T 2C -B test verify -Pflink1 -pl "${TEST_MODULE}" -Duser.timezone=$jvm_timezone + mvn -T 2C -B test verify -Pflink1,spark3 -pl "${TEST_MODULE}" -Duser.timezone=$jvm_timezone echo "All modules tested" env: MAVEN_OPTS: -Xmx4096m -XX:+UseG1GC -XX:CICompilerCount=2 \ No newline at end of file diff --git a/.github/workflows/utitcase-flink-2.x-jdk11.yml b/.github/workflows/utitcase-flink-2.x-jdk11.yml index 44a0b42ad98b..62be40d74d2b 100644 --- a/.github/workflows/utitcase-flink-2.x-jdk11.yml +++ b/.github/workflows/utitcase-flink-2.x-jdk11.yml @@ -41,7 +41,9 @@ jobs: java-version: ${{ env.JDK_VERSION }} distribution: 'temurin' - name: Build Flink - run: mvn -T 1C -B clean install -DskipTests -Pflink2 + run: | + mvn -T 1C -B clean install -DskipTests -Pflink1,spark3 + mvn -T 1C -B clean install -DskipTests -Pflink2,spark3 - name: Test Flink run: | # run tests with random timezone to find out timezone related bugs @@ -53,6 +55,6 @@ jobs: test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" - mvn -T 1C -B clean install -Pflink2 -pl "${test_modules}" -Duser.timezone=$jvm_timezone + mvn -T 1C -B clean install -Pflink2,spark3 -pl "${test_modules}" -Duser.timezone=$jvm_timezone env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/utitcase-spark-4.x.yml b/.github/workflows/utitcase-spark-4.x.yml index ff6f95d4153d..df1be7bd2c0e 100644 --- a/.github/workflows/utitcase-spark-4.x.yml +++ b/.github/workflows/utitcase-spark-4.x.yml @@ -45,7 +45,7 @@ jobs: java-version: ${{ env.JDK_VERSION }} distribution: 'temurin' - name: Build Spark - run: mvn -T 2C -B clean install -DskipTests -Pspark4 + run: mvn -T 2C -B clean install -DskipTests -Pspark4,flink1 - name: Test Spark timeout-minutes: 60 run: | @@ -58,6 +58,6 @@ jobs: test_modules+="org.apache.paimon:paimon-spark-${suffix}," done test_modules="${test_modules%,}" - mvn -T 2C -B test -pl "${test_modules}" -Duser.timezone=$jvm_timezone -Pspark4 + mvn -T 2C -B test -pl "${test_modules}" -Duser.timezone=$jvm_timezone -Pspark4,flink1 env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/paimon-e2e-tests/README.md b/paimon-e2e-tests/README.md index 31147cc33a8c..8d968ff41602 100644 --- a/paimon-e2e-tests/README.md +++ b/paimon-e2e-tests/README.md @@ -32,11 +32,11 @@ Currently, e2e supports docker environment only. You need to do some preparation 1. Build with Flink-versioned profiles, like flink-1.17. The default Flink main version is declared in the parent `pom.xml` by `` tag. ```bash -mvn clean install -DskipTests -Pflink-1.17 +mvn clean install -DskipTests -Pflink1,flink-1.17,spark3 ``` 2. Run e2e tests ```bash -mvn test -pl paimon-e2e-tests -Pflink-1.17 +mvn test -pl paimon-e2e-tests -Pflink1,flink-1.17,spark3 ``` \ No newline at end of file diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh index 0622eff1bbc2..cddff9ad91be 100755 --- a/tools/releasing/deploy_staging_jars.sh +++ b/tools/releasing/deploy_staging_jars.sh @@ -42,6 +42,6 @@ fi cd ${PROJECT_ROOT} echo "Deploying to repository.apache.org" -${MVN} clean deploy -Papache-release,docs-and-source,spark3 -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS +${MVN} clean deploy -Papache-release,docs-and-source,spark3,flink1 -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS cd ${CURR_DIR}