[BEAM-13430] Remove propdeps and replace with compileOnly

buildSrc/build.gradle.kts

@@ -43,7 +43,6 @@ dependencies {
 
   runtimeOnly("net.ltgt.gradle:gradle-apt-plugin:0.21")                                                    // Enable a Java annotation processor
   runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13")                                          // Enable proto code generation
-  runtimeOnly("io.spring.gradle:propdeps-plugin:0.0.9.RELEASE")                                            // Enable provided and optional configurations
   runtimeOnly("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0")                                   // Enable Avro code generation
   runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1")                                       // Enable a code formatting plugin
   runtimeOnly("gradle.plugin.com.github.blindpirate:gogradle:0.11.4")                                      // Enable Go code compilation

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -748,9 +748,6 @@ class BeamModulePlugin implements Plugin<Project> {
     //  * java
     //  * maven
     //  * net.ltgt.apt (plugin to configure annotation processing tool)
-    //  * propdeps (provide optional and provided dependency configurations)
-    //  * propdeps-maven
-    //  * propdeps-idea
     //  * checkstyle
     //  * spotbugs
     //  * shadow (conditional on shadowClosure being specified)
@@ -1032,13 +1029,6 @@ class BeamModulePlugin implements Plugin<Project> {
         implementation "org.checkerframework:checker-qual:$checkerframework_version"
       }
 
-      // Add the optional and provided configurations for dependencies
-      // TODO: Either remove these plugins and find another way to generate the Maven poms
-      // with the correct dependency scopes configured.
-      project.apply plugin: 'propdeps'
-      project.apply plugin: 'propdeps-maven'
-      project.apply plugin: 'propdeps-idea'
-
       // Defines Targets for sonarqube analysis reporting.
       project.apply plugin: "org.sonarqube"
 
@@ -1254,7 +1244,7 @@ class BeamModulePlugin implements Plugin<Project> {
 
           project.dependencies {
             shadowTestRuntimeClasspath it.project(path: project.path, configuration: "shadowTest")
-            shadowTestRuntimeClasspath it.project(path: project.path, configuration: "provided")
+            shadowTestRuntimeClasspath it.project(path: project.path, configuration: "compileOnly")
           }
 
           project.test { classpath = project.configurations.shadowTestRuntimeClasspath }
@@ -1580,8 +1570,8 @@ class BeamModulePlugin implements Plugin<Project> {
                 // TODO: Should we use the runtime scope instead of the compile scope
                 // which forces all our consumers to declare what they consume?
                 generateDependenciesFromConfiguration(
-                    configuration: (configuration.shadowClosure ? 'shadow' : 'compile'), scope: 'compile')
-                generateDependenciesFromConfiguration(configuration: 'provided', scope: 'provided')
+                    configuration: (configuration.shadowClosure ? 'shadow' : 'implementation'), scope: 'compile')
+                generateDependenciesFromConfiguration(configuration: 'compileOnly', scope: 'provided')
 
                 if (!boms.isEmpty()) {
                   def dependencyManagementNode = root.appendNode('dependencyManagement')

examples/java/build.gradle

@@ -77,8 +77,8 @@ dependencies {
   implementation library.java.proto_google_cloud_bigtable_v2
   implementation library.java.proto_google_cloud_datastore_v1
   implementation library.java.slf4j_api
-  provided library.java.commons_io
-  provided library.java.commons_csv
+  implementation library.java.commons_io
+  implementation library.java.commons_csv
   runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   implementation library.java.vendored_grpc_1_36_0
   implementation library.java.vendored_guava_26_0_jre

examples/java/twitter/build.gradle

@@ -50,8 +50,8 @@ dependencies {
     implementation project(path: ":sdks:java:core", configuration: "shadow")
     implementation library.java.joda_time
     implementation library.java.slf4j_api
-    provided library.java.commons_io
-    provided library.java.commons_csv
+    implementation library.java.commons_io
+    implementation library.java.commons_csv
     runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
     implementation ("org.twitter4j:twitter4j-stream:4.0.7")
     implementation ("org.twitter4j:twitter4j-core:4.0.7")

examples/kotlin/build.gradle

@@ -52,7 +52,7 @@ dependencies {
   implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
   implementation library.java.vendored_guava_26_0_jre
   implementation project(path: ":sdks:java:core", configuration: "shadow")
-  // Add the dependency that sdks:java:core that is marked as provided
+  // Add the dependency that sdks:java:core that is marked as compileOnly
   implementation library.java.hamcrest_core
   implementation project(":sdks:java:extensions:google-cloud-platform-core")
   implementation project(":sdks:java:io:google-cloud-platform")

ownership/JAVA_DEPENDENCY_OWNERS.yaml

@@ -514,11 +514,6 @@ deps:
     artifact: opencensus-contrib-grpc-metrics
     owners:
 
-  io.spring.gradle:propdeps-plugin:
-    group: io.spring.gradle
-    artifact: propdeps-plugin
-    owners:
-
   javax.xml.bind:jaxb-api:
     group: javax.xml.bind
     artifact: jaxb-api

runners/direct-java/build.gradle

@@ -74,8 +74,8 @@ dependencies {
   shadow library.java.joda_time
   shadow library.java.slf4j_api
   shadow library.java.jackson_databind
-  provided library.java.hamcrest_core
-  provided library.java.junit
+  compileOnly library.java.hamcrest_core
+  compileOnly library.java.junit
   shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
   shadowTest project(path: ":runners:core-java", configuration: "testRuntimeMigration")
   shadowTest library.java.mockito_core

runners/flink/flink_runner.gradle

@@ -148,7 +148,8 @@ dependencies {
   implementation "org.apache.flink:flink-runtime_2.11:$flink_version"
   implementation "org.apache.flink:flink-streaming-java_2.11:$flink_version"
   // RocksDB state backend (included in the Flink distribution)
-  provided "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
+  compileOnly "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
+  testImplementation "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   // FlinkStateInternalsTest extends abstract StateInternalsTest
   testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration")

runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle

@@ -185,17 +185,22 @@ compileTestJava {
 }
 
 dependencies {
+    implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+
     // Note that any dependency that is modified here should also be modified within
     // runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided
     // there.
     sdk_provided_dependencies.each {
-        provided(it)
+        compileOnly(it)
+        testImplementation(it)
     }
     sdk_provided_shaded_project_dependencies.each {
-        provided project(path: it, configuration: "shadow")
+        compileOnly project(path: it, configuration: "shadow")
+        testImplementation project(path: it, configuration: "shadow")
     }
     sdk_provided_project_dependencies.each {
-        provided project(it)
+        compileOnly project(path: it)
+        testImplementation project(path: it)
     }
 
     implementation project(path: ":model:fn-execution", configuration: "shadow")
@@ -208,6 +213,9 @@ dependencies {
     implementation project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow")
     shadow library.java.vendored_guava_26_0_jre
 
+    implementation library.java.google_auth_library_credentials
+    implementation library.java.proto_google_common_protos
+
     // Conscrypt shouldn't be included here because Conscrypt won't work when being shaded.
     // (Context: https://github.com/apache/beam/pull/13846)
     // Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now.
@@ -223,7 +231,7 @@ dependencies {
     implementation library.java.joda_time
     implementation library.java.slf4j_api
     implementation library.java.vendored_grpc_1_36_0
-    provided library.java.error_prone_annotations
+    implementation library.java.error_prone_annotations
     runtimeOnly library.java.slf4j_jdk14
 
     // Any test dependency which intersects with our relocation rules above needs to be relocated

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java

@@ -20,7 +20,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.json.gson.GsonFactory;
 import com.google.api.client.util.Key;
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.GetDebugConfigRequest;
@@ -50,7 +50,7 @@
 })
 public class DebugCapture {
   private static final Logger LOG = LoggerFactory.getLogger(DebugCapture.class);
-  private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+  private static final JsonFactory JSON_FACTORY = new GsonFactory();
   private static final String COMPONENT = "JavaHarness";
   // How often to refresh debug capture config.
   private static final long UPDATE_CONFIG_PERIOD_SEC = 60;

runners/java-fn-execution/build.gradle

@@ -25,7 +25,8 @@ description = "Apache Beam :: Runners :: Java Fn Execution"
 dependencies {
   implementation library.java.vendored_guava_26_0_jre
   implementation project(":runners:core-construction-java")
-  provided project(":sdks:java:harness")
+  implementation project(":runners:core-java")
+  compileOnly project(":sdks:java:harness")
   implementation project(path: ":model:pipeline", configuration: "shadow")
   implementation project(path: ":model:fn-execution", configuration: "shadow")
   implementation project(path: ":sdks:java:core", configuration: "shadow")
@@ -37,8 +38,8 @@ dependencies {
   implementation library.java.joda_time
   testImplementation project(":sdks:java:harness")
   testImplementation project(":runners:core-construction-java")
-  testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration")
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
+  testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration")
   testImplementation library.java.junit
   testImplementation library.java.mockito_core
   testRuntimeOnly library.java.slf4j_simple

runners/spark/job-server/spark_job_server.gradle

@@ -52,9 +52,9 @@ configurations.all {
 dependencies {
   implementation project(sparkRunnerProject)
   permitUnusedDeclared project(sparkRunnerProject)
-  implementation project(path: sparkRunnerProject, configuration: "provided")
+  implementation project(path: sparkRunnerProject, configuration: "compileOnly")
   validatesPortableRunner project(path: sparkRunnerProject, configuration: "testRuntimeMigration")
-  validatesPortableRunner project(path: sparkRunnerProject, configuration: "provided")
+  validatesPortableRunner project(path: sparkRunnerProject, configuration: "compileOnly")
   validatesPortableRunner project(path: ":sdks:java:core", configuration: "shadowTest")
   validatesPortableRunner project(path: ":runners:core-java", configuration: "testRuntimeMigration")
   validatesPortableRunner project(path: ":runners:portability:java", configuration: "testRuntimeMigration")

runners/spark/spark_runner.gradle

@@ -162,22 +162,31 @@ dependencies {
   implementation project(":sdks:java:fn-execution")
   implementation library.java.vendored_grpc_1_36_0
   implementation library.java.vendored_guava_26_0_jre
-  provided "org.apache.spark:spark-core_$spark_scala_version:$spark_version"
-  provided "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
-  provided "org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
-  provided "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
+  compileOnly "org.apache.spark:spark-core_$spark_scala_version:$spark_version"
+  compileOnly "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
+  compileOnly "org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
+  compileOnly "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
   if(project.property("spark_scala_version").equals("2.11")){
     runtimeOnly library.java.jackson_module_scala_2_11
   } else {
     runtimeOnly library.java.jackson_module_scala_2_12
   }
   // Force paranamer 2.8 to avoid issues when using Scala 2.12
   runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8"
-  provided library.java.hadoop_common
-  provided library.java.commons_io
-  provided library.java.hamcrest_core
-  provided library.java.hamcrest_library
-  provided "com.esotericsoftware.kryo:kryo:2.21"
+  compileOnly library.java.hadoop_common
+  compileOnly library.java.commons_io
+  compileOnly library.java.hamcrest_core
+  compileOnly library.java.hamcrest_library
+  compileOnly "com.esotericsoftware.kryo:kryo:2.21"
+  testImplementation "org.apache.spark:spark-core_$spark_scala_version:$spark_version"
+  testImplementation "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version"
+  testImplementation "org.apache.spark:spark-sql_$spark_scala_version:$spark_version"
+  testImplementation "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version"
+  testImplementation library.java.hadoop_common
+  testImplementation library.java.commons_io
+  testImplementation library.java.hamcrest_core
+  testImplementation library.java.hamcrest_library
+  testImplementation "com.esotericsoftware.kryo:kryo:2.21"
   testImplementation project(":sdks:java:io:kafka")
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   // SparkStateInternalsTest extends abstract StateInternalsTest
@@ -196,7 +205,7 @@ dependencies {
   validatesRunner project(path: ":examples:java", configuration: "testRuntimeMigration")
   validatesRunner project(path: project.path, configuration: "testRuntimeMigration")
   validatesRunner project(project.path)
-  validatesRunner project(path: project.path, configuration: "provided")
+  validatesRunner project(path: project.path, configuration: "compileOnly")
   hadoopVersions.each {kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
   }

sdks/java/container/build.gradle

@@ -40,7 +40,7 @@ dependencies {
   dockerDependency project(path: ":sdks:java:harness", configuration: "shadow")
   // For executing KafkaIO, e.g. as an external transform
   dockerDependency project(":sdks:java:io:kafka")
-  // This dependency is set to 'provided' scope in :sdks:java:io:kafka
+  // This dependency is set to 'compileOnly' scope in :sdks:java:io:kafka
   dockerDependency library.java.kafka_clients
 }
 

sdks/java/core/build.gradle

@@ -90,11 +90,11 @@ dependencies {
   shadow library.java.avro
   shadow library.java.snappy_java
   shadow library.java.joda_time
-  provided library.java.junit
-  provided library.java.hamcrest_core
-  provided library.java.hamcrest_library
-  provided 'io.airlift:aircompressor:0.18'
-  provided 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
+  compileOnly library.java.junit
+  compileOnly library.java.hamcrest_core
+  compileOnly library.java.hamcrest_library
+  compileOnly 'io.airlift:aircompressor:0.18'
+  compileOnly 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
   shadowTest library.java.jackson_dataformat_yaml
   shadowTest library.java.guava_testlib
   shadowTest library.java.mockito_core
@@ -105,4 +105,6 @@ dependencies {
   shadowTest library.java.avro_tests
   shadowTest library.java.zstd_jni
   testRuntimeOnly library.java.slf4j_jdk14
+  testImplementation 'io.airlift:aircompressor:0.18'
+  testImplementation 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
 }

sdks/java/extensions/google-cloud-platform-core/build.gradle

@@ -55,8 +55,8 @@ dependencies {
   implementation library.java.jackson_annotations
   implementation library.java.jackson_databind
   permitUnusedDeclared library.java.jackson_databind // BEAM-11761
-  provided library.java.hamcrest_core
-  provided library.java.junit
+  compileOnly library.java.hamcrest_core
+  compileOnly library.java.junit
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   testImplementation library.java.mockito_core
   testRuntimeOnly library.java.slf4j_jdk14

sdks/java/extensions/ml/build.gradle

@@ -48,7 +48,7 @@ dependencies {
     implementation library.java.gax
     implementation library.java.protobuf_java
     implementation library.java.slf4j_api
-    provided library.java.junit
+    compileOnly library.java.junit
     testImplementation project(path: ':sdks:java:core', configuration: 'shadowTest')
     implementation 'com.google.cloud:google-cloud-vision:1.99.3'
     testImplementation library.java.mockito_core

sdks/java/extensions/sorter/build.gradle

@@ -36,10 +36,12 @@ dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation library.java.vendored_guava_26_0_jre
   implementation library.java.slf4j_api
-  provided library.java.hadoop_mapreduce_client_core
-  provided library.java.hadoop_common
+  compileOnly library.java.hadoop_mapreduce_client_core
+  compileOnly library.java.hadoop_common
   testImplementation library.java.mockito_core
   testImplementation library.java.junit
+  testImplementation library.java.hadoop_mapreduce_client_core
+  testImplementation library.java.hadoop_common
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   hadoopVersions.each{kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"