From cccbc5073672c67686905fd5d170db4239440c83 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 14 Jul 2025 13:39:10 -0400 Subject: [PATCH 1/2] Bump gradle plugin versions previously blocked by Java8 support Now we dropped Java8 support to build Beam, we can bump the following plugins * Shadow plugin to 8.3.8 * grgit-gradle to 5.3.2 * Remove Java8 logic for Errorprone --- buildSrc/build.gradle.kts | 8 +++++--- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 ++-------- .../org/apache/beam/gradle/VendorJavaPlugin.groovy | 2 +- examples/java/build.gradle | 2 +- examples/java/cdap/hubspot/build.gradle | 2 +- examples/java/cdap/salesforce/build.gradle | 2 +- examples/java/cdap/servicenow/build.gradle | 2 +- examples/java/cdap/zendesk/build.gradle | 2 +- examples/java/sql/build.gradle | 2 +- examples/java/twitter/build.gradle | 2 +- .../notebooks/get-started/try-apache-beam-java.ipynb | 2 +- .../python-custom-multi-language-pipelines-guide.md | 2 +- 12 files changed, 17 insertions(+), 21 deletions(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index d2f89cc62325..d053713cab91 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -41,18 +41,20 @@ repositories { dependencies { implementation(gradleApi()) implementation(localGroovy()) - implementation("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") + implementation("com.gradleup.shadow:shadow-gradle-plugin:8.3.8") { // Enable shading Java dependencies + exclude(group="org.codehaus.plexus", module="plexus-xml") // plexus-xml 4.x requires Java17 + } + runtimeOnly("org.codehaus.plexus:plexus-xml:3.0.2") implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14") runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation. Version 1.1.0 is the last supporting avro 1.10.2 runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree - runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5") runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:3.1.0") // Enable errorprone Java static analysis - runtimeOnly("org.ajoberstar.grgit:grgit-gradle:4.1.1") // Enable website git publish to asf-site branch + runtimeOnly("org.ajoberstar.grgit:grgit-gradle:5.3.2") // Enable website git publish to asf-site branch runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.16.12") // Enable docker compose tasks runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3") // Enable dep analysis runtimeOnly("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11") // Enable dep vulnerability analysis diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 52fcd2a63a88..be139b3293be 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1495,11 +1495,6 @@ class BeamModulePlugin implements Plugin { project.dependencies { errorprone("com.google.errorprone:error_prone_core:$errorprone_version") errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2") - // At least JDk 9 compiler is required, however JDK 8 still can be used but with additional errorproneJavac - // configuration. For more details please see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) == 0) { - errorproneJavac("com.google.errorprone:javac:9+181-r4173-1") - } } project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } @@ -1512,8 +1507,7 @@ class BeamModulePlugin implements Plugin { // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. // The -J prefix is not needed if forkOptions.javaHome is unset, // see http://github.com/gradle/gradle/issues/22747 - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 - && options.forkOptions.javaHome == null) { + if (options.forkOptions.javaHome == null) { options.fork = true options.forkOptions.jvmArgs += errorProneAddModuleOpts } @@ -1633,7 +1627,7 @@ class BeamModulePlugin implements Plugin { // // TODO: Enforce all relocations are always performed to: // getJavaRelocatedPath(package_suffix) where package_suffix is something like "com.google.commmon" - project.apply plugin: 'com.github.johnrengelman.shadow' + project.apply plugin: 'com.gradleup.shadow' // Create a new configuration 'shadowTest' like 'shadow' for the test scope project.configurations { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy index 82e85d0ed468..f6a581b4fc68 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/VendorJavaPlugin.groovy @@ -80,7 +80,7 @@ class VendorJavaPlugin implements Plugin { config.version += '-SNAPSHOT' } - project.apply plugin: 'com.github.johnrengelman.shadow' + project.apply plugin: 'com.gradleup.shadow' project.apply plugin: 'java' diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 98763e911134..5a1d5b2e8fdc 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/cdap/hubspot/build.gradle b/examples/java/cdap/hubspot/build.gradle index 94ff682aa2ee..09d9a6a378e2 100644 --- a/examples/java/cdap/hubspot/build.gradle +++ b/examples/java/cdap/hubspot/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/cdap/salesforce/build.gradle b/examples/java/cdap/salesforce/build.gradle index a6fc344e0008..cc14c61aeb59 100644 --- a/examples/java/cdap/salesforce/build.gradle +++ b/examples/java/cdap/salesforce/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/cdap/servicenow/build.gradle b/examples/java/cdap/servicenow/build.gradle index f113677bae53..1560956b7195 100644 --- a/examples/java/cdap/servicenow/build.gradle +++ b/examples/java/cdap/servicenow/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/cdap/zendesk/build.gradle b/examples/java/cdap/zendesk/build.gradle index ef99fe8306ed..489e06128e38 100644 --- a/examples/java/cdap/zendesk/build.gradle +++ b/examples/java/cdap/zendesk/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/sql/build.gradle b/examples/java/sql/build.gradle index 466e2d0f429d..05bfaec2a80a 100644 --- a/examples/java/sql/build.gradle +++ b/examples/java/sql/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/java/twitter/build.gradle b/examples/java/twitter/build.gradle index 27f72c310e6e..34ac3ef9e94f 100644 --- a/examples/java/twitter/build.gradle +++ b/examples/java/twitter/build.gradle @@ -21,7 +21,7 @@ import groovy.json.JsonOutput plugins { id 'java' id 'org.apache.beam.module' - id 'com.github.johnrengelman.shadow' + id 'com.gradleup.shadow' } applyJavaNature( diff --git a/examples/notebooks/get-started/try-apache-beam-java.ipynb b/examples/notebooks/get-started/try-apache-beam-java.ipynb index b19030177f28..46ab413083cf 100644 --- a/examples/notebooks/get-started/try-apache-beam-java.ipynb +++ b/examples/notebooks/get-started/try-apache-beam-java.ipynb @@ -298,7 +298,7 @@ " id 'application'\n", "\n", " // 'shadow' allows us to embed all the dependencies into a fat jar.\n", - " id 'com.github.johnrengelman.shadow' version '4.0.3'\n", + " id 'com.gradleup.shadow' version '8.3.8'\n", "}\n", "\n", "// This is the path of the main class, stored within ./src/main/java/\n", diff --git a/website/www/site/content/en/documentation/sdks/python-custom-multi-language-pipelines-guide.md b/website/www/site/content/en/documentation/sdks/python-custom-multi-language-pipelines-guide.md index 18afe8aafa08..8b5fe5ad95e4 100644 --- a/website/www/site/content/en/documentation/sdks/python-custom-multi-language-pipelines-guide.md +++ b/website/www/site/content/en/documentation/sdks/python-custom-multi-language-pipelines-guide.md @@ -164,7 +164,7 @@ First, build a shaded JAR file that includes: ### Gradle build file ```groovy plugins { - id 'com.github.johnrengelman.shadow' version '8.1.1' + id 'com.gradleup.shadow' version '8.3.8' id 'application' } From 61cf06eaf90d6c64d93985b789dd8c74100d14fd Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 14 Jul 2025 16:21:56 -0400 Subject: [PATCH 2/2] Clean up docs --- .../beam_PreCommit_Java_Examples_Dataflow_Java21.yml | 4 ++-- .github/workflows/beam_PreCommit_SQL_Java17.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml index fe8efc3ac344..aa278822550c 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml @@ -93,8 +93,8 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - # The test requires Java 21 and Java 8 versions. - # Java 8 is installed second because JAVA_HOME needs to point to Java 8. + # The test requires Java 21 and Java 11 versions. + # Java 11 is installed second because JAVA_HOME needs to point to Java 11. - name: Setup environment uses: ./.github/actions/setup-environment-action with: diff --git a/.github/workflows/beam_PreCommit_SQL_Java17.yml b/.github/workflows/beam_PreCommit_SQL_Java17.yml index 2ff54aad6355..aaa0f51d1bdb 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java17.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java17.yml @@ -78,8 +78,8 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) -# The test requires Java 17 and Java 8 versions. -# Java 8 is installed second because JAVA_HOME needs to point to Java 8. +# The test requires Java 17 and Java 11 versions. +# Java 11 is installed second because JAVA_HOME needs to point to Java 11. - name: Setup environment uses: ./.github/actions/setup-environment-action with: