[BEAM-13430] Re-add provided configuration

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -807,6 +807,12 @@ class BeamModulePlugin implements Plugin<Project> {
         testRuntimeMigration.extendsFrom(testRuntimeOnly)
       }
 
+      // Provided configuration to match Maven provided scope
+      project.configurations {
+        provided
+        implementation.extendsFrom(provided)
+      }
+
       // Configure the Java compiler source language and target compatibility levels. Also ensure that
       // we configure the Java compiler to use UTF-8.
       project.sourceCompatibility = project.javaVersion
@@ -1544,7 +1550,7 @@ class BeamModulePlugin implements Plugin<Project> {
                 // which forces all our consumers to declare what they consume?
                 generateDependenciesFromConfiguration(
                     configuration: (configuration.shadowClosure ? 'shadow' : 'implementation'), scope: 'compile')
-                generateDependenciesFromConfiguration(configuration: 'compileOnly', scope: 'provided')
+                generateDependenciesFromConfiguration(configuration: 'provided', scope: 'provided')
 
                 if (!boms.isEmpty()) {
                   def dependencyManagementNode = root.appendNode('dependencyManagement')

examples/kotlin/build.gradle

@@ -52,7 +52,7 @@ dependencies {
   implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
   implementation library.java.vendored_guava_26_0_jre
   implementation project(path: ":sdks:java:core", configuration: "shadow")
-  // Add the dependency that sdks:java:core that is marked as compileOnly
+  // Add the dependency that sdks:java:core that is marked as provided
   implementation library.java.hamcrest
   permitUnusedDeclared library.java.hamcrest
   implementation project(":sdks:java:extensions:google-cloud-platform-core")

runners/direct-java/build.gradle

@@ -76,9 +76,9 @@ dependencies {
   shadow library.java.joda_time
   shadow library.java.slf4j_api
   shadow library.java.jackson_databind
-  compileOnly library.java.hamcrest
+  provided library.java.hamcrest
   permitUnusedDeclared library.java.hamcrest
-  compileOnly library.java.junit
+  provided library.java.junit
   permitUnusedDeclared library.java.junit
   shadowTest project(path: ":sdks:java:core", configuration: "shadowTest")
   shadowTest project(path: ":runners:core-java", configuration: "testRuntimeMigration")

runners/flink/flink_runner.gradle

@@ -150,8 +150,7 @@ dependencies {
   implementation "org.apache.flink:flink-runtime_2.11:$flink_version"
   implementation "org.apache.flink:flink-streaming-java_2.11:$flink_version"
   // RocksDB state backend (included in the Flink distribution)
-  compileOnly "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
-  testImplementation "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
+  provided "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version"
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   // FlinkStateInternalsTest extends abstract StateInternalsTest
   testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration")

runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle

@@ -189,16 +189,13 @@ dependencies {
     // runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided
     // there.
     sdk_provided_dependencies.each {
-        compileOnly(it)
-        testImplementation(it)
+        provided(it)
     }
     sdk_provided_shaded_project_dependencies.each {
-        compileOnly project(path: it, configuration: "shadow")
-        testImplementation project(path: it, configuration: "shadow")
+        provided project(path: it, configuration: "shadow")
     }
     sdk_provided_project_dependencies.each {
-        compileOnly project(path: it)
-        testImplementation project(path: it)
+        provided project(path: it)
     }
 
     implementation project(path: ":model:fn-execution", configuration: "shadow")

runners/portability/java/build.gradle

@@ -36,6 +36,7 @@ dependencies {
   implementation project(":runners:java-job-service")
   implementation project(path: ":sdks:java:harness", configuration: "shadow")
   implementation library.java.hamcrest
+  permitUnusedDeclared library.java.hamcrest
 
   testImplementation project(path: ":runners:core-construction-java", configuration: "testRuntimeMigration")
   testImplementation library.java.hamcrest

runners/spark/spark_runner.gradle

@@ -183,14 +183,10 @@ dependencies {
   }
   // Force paranamer 2.8 to avoid issues when using Scala 2.12
   runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8"
-  compileOnly library.java.hadoop_common
-  compileOnly library.java.commons_io
-  compileOnly library.java.hamcrest
-  compileOnly "com.esotericsoftware:kryo-shaded:4.0.2"
-  testImplementation library.java.hadoop_common
-  testImplementation library.java.commons_io
-  testImplementation library.java.hamcrest
-  testImplementation "com.esotericsoftware.kryo:kryo:2.21"
+  provided library.java.hadoop_common
+  provided library.java.commons_io
+  provided library.java.hamcrest
+  provided "com.esotericsoftware:kryo-shaded:4.0.2"
   testImplementation project(":sdks:java:io:kafka")
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   // SparkStateInternalsTest extends abstract StateInternalsTest

sdks/java/container/build.gradle

@@ -40,7 +40,7 @@ dependencies {
   dockerDependency project(path: ":sdks:java:harness", configuration: "shadow")
   // For executing KafkaIO, e.g. as an external transform
   dockerDependency project(":sdks:java:io:kafka")
-  // This dependency is set to 'compileOnly' scope in :sdks:java:io:kafka
+  // This dependency is set to 'provided' scope in :sdks:java:io:kafka
   dockerDependency library.java.kafka_clients
   dockerDependency library.java.jamm
 }

sdks/java/container/common.gradle

@@ -46,12 +46,14 @@ dependencies {
 
 task copyDockerfileDependencies(type: Copy) {
     from configurations.dockerDependency
-    rename "slf4j-api.*", "slf4j-api.jar"
-    rename "slf4j-jdk14.*", "slf4j-jdk14.jar"
+    rename 'slf4j-api.*', 'slf4j-api.jar'
+    rename 'slf4j-jdk14.*', 'slf4j-jdk14.jar'
     rename 'beam-sdks-java-harness-.*.jar', 'beam-sdks-java-harness.jar'
     rename 'beam-sdks-java-io-kafka.*.jar', 'beam-sdks-java-io-kafka.jar'
     rename 'kafka-clients.*.jar', 'kafka-clients.jar'
-    rename "jamm.*.jar", "jamm.jar"
+    rename 'jamm.*.jar', 'jamm.jar'
+
+    setDuplicatesStrategy(DuplicatesStrategy.INCLUDE)
     into "build/target"
 }
 

sdks/java/core/build.gradle

@@ -90,10 +90,10 @@ dependencies {
   shadow library.java.avro
   shadow library.java.snappy_java
   shadow library.java.joda_time
-  compileOnly library.java.junit
-  compileOnly library.java.hamcrest
-  compileOnly 'io.airlift:aircompressor:0.18'
-  compileOnly 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
+  provided library.java.junit
+  provided library.java.hamcrest
+  provided 'io.airlift:aircompressor:0.18'
+  provided 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
   permitUnusedDeclared 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
   shadowTest library.java.jackson_dataformat_yaml
   shadowTest library.java.guava_testlib
@@ -105,6 +105,4 @@ dependencies {
   shadowTest library.java.zstd_jni
   shadowTest library.java.jamm
   testRuntimeOnly library.java.slf4j_jdk14
-  testImplementation 'io.airlift:aircompressor:0.18'
-  testImplementation 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1'
 }

sdks/java/extensions/sorter/build.gradle

@@ -36,13 +36,11 @@ dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation library.java.vendored_guava_26_0_jre
   implementation library.java.slf4j_api
-  compileOnly library.java.hadoop_mapreduce_client_core
-  compileOnly library.java.hadoop_common
+  provided library.java.hadoop_mapreduce_client_core
+  provided library.java.hadoop_common
   testImplementation library.java.mockito_core
   testImplementation library.java.junit
   testImplementation library.java.hamcrest
-  testImplementation library.java.hadoop_mapreduce_client_core
-  testImplementation library.java.hadoop_common
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   hadoopVersions.each{kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"

sdks/java/extensions/sql/build.gradle

@@ -67,7 +67,7 @@ dependencies {
   implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
 
   // TODO(BEAM-13179): Determine how to build without this dependency
-  compileOnly "org.immutables:value:2.8.8"
+  provided "org.immutables:value:2.8.8"
   permitUnusedDeclared "org.immutables:value:2.8.8"
   javacc "net.java.dev.javacc:javacc:4.0"
   fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
@@ -91,7 +91,7 @@ dependencies {
   implementation library.java.slf4j_api
   implementation library.java.joda_time
   implementation library.java.vendored_guava_26_0_jre
-  compileOnly project(":sdks:java:io:kafka")
+  provided project(":sdks:java:io:kafka")
   implementation project(":sdks:java:extensions:google-cloud-platform-core")
   permitUnusedDeclared project(":sdks:java:extensions:google-cloud-platform-core")
   implementation project(":sdks:java:io:google-cloud-platform")
@@ -104,12 +104,12 @@ dependencies {
   implementation library.java.avro
   implementation library.java.protobuf_java
   implementation library.java.protobuf_java_util
-  compileOnly project(":sdks:java:io:parquet")
-  compileOnly library.java.jackson_dataformat_xml
+  provided project(":sdks:java:io:parquet")
+  provided library.java.jackson_dataformat_xml
   permitUnusedDeclared library.java.jackson_dataformat_xml
-  compileOnly library.java.hadoop_client
+  provided library.java.hadoop_client
   permitUnusedDeclared library.java.hadoop_client
-  compileOnly library.java.kafka_clients
+  provided library.java.kafka_clients
   testImplementation library.java.vendored_calcite_1_28_0
   testImplementation library.java.vendored_guava_26_0_jre
   testImplementation library.java.junit
@@ -120,10 +120,6 @@ dependencies {
   testImplementation library.java.proto_google_cloud_bigtable_admin_v2
   testImplementation library.java.proto_google_cloud_datastore_v1
   testImplementation library.java.google_cloud_datastore_v1_proto_client
-  testImplementation library.java.hadoop_client
-  testImplementation library.java.kafka_clients
-  testImplementation project(":sdks:java:io:kafka")
-  testImplementation project(":sdks:java:io:parquet")
   testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration")
   testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration")
   testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")

sdks/java/extensions/sql/datacatalog/build.gradle

@@ -38,18 +38,16 @@ dependencies {
   implementation "com.alibaba:fastjson:1.2.69"
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation "org.threeten:threetenbp:1.4.5"
-  compileOnly project(":sdks:java:extensions:sql")
+  provided project(":sdks:java:extensions:sql")
 
   // Dependencies for the example
   implementation project(":sdks:java:extensions:google-cloud-platform-core")
-  compileOnly project(":sdks:java:io:google-cloud-platform")
+  provided project(":sdks:java:io:google-cloud-platform")
   permitUnusedDeclared project(":sdks:java:io:google-cloud-platform")
   implementation library.java.slf4j_api
 
-  testImplementation project(":sdks:java:extensions:sql")
   testImplementation project(":sdks:java:extensions:sql:zetasql")
   testImplementation project(":runners:direct-java")
-  testImplementation project(":sdks:java:io:google-cloud-platform")
   testImplementation library.java.google_api_services_bigquery
   testImplementation library.java.junit
   testRuntimeOnly library.java.slf4j_simple

sdks/java/extensions/sql/zetasql/build.gradle

@@ -31,7 +31,7 @@ def zetasql_version = "2020.10.1"
 
 dependencies {
   // TODO(BEAM-13179): Determine how to build without this dependency
-  compileOnly "org.immutables:value:2.8.8"
+  provided "org.immutables:value:2.8.8"
   permitUnusedDeclared "org.immutables:value:2.8.8"
   implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
   permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom)

sdks/java/fn-execution/build.gradle

@@ -33,7 +33,7 @@ dependencies {
   implementation library.java.vendored_guava_26_0_jre
   implementation library.java.slf4j_api
   implementation library.java.joda_time
-  compileOnly library.java.junit
+  provided library.java.junit
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   testImplementation library.java.junit
   testImplementation library.java.mockito_core

sdks/java/harness/build.gradle

@@ -75,7 +75,7 @@ dependencies {
   testImplementation project(":runners:core-construction-java")
   testImplementation project(path: ":sdks:java:fn-execution", configuration: "testRuntimeMigration")
   shadowTestRuntimeClasspath library.java.slf4j_jdk14
-  jmhImplementation project(path: ":sdks:java:harness", configuration: "shadowTest")
+  jmhImplementation project(path: ":sdks:java:harness", configuration: "shadow")
   jmhImplementation project(":runners:java-fn-execution")
   jmhRuntimeOnly library.java.slf4j_jdk14
 }

sdks/java/io/debezium/build.gradle

@@ -36,7 +36,7 @@ dependencies {
     implementation project(path: ":sdks:java:core", configuration: "shadow")
     implementation library.java.slf4j_api
     implementation library.java.joda_time
-    compileOnly library.java.jackson_dataformat_csv
+    provided library.java.jackson_dataformat_csv
     permitUnusedDeclared library.java.jackson_dataformat_csv
     testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
     testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration")

sdks/java/io/hadoop-common/build.gradle

@@ -35,14 +35,11 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
 
 dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
-  compileOnly library.java.hadoop_client
+  provided library.java.hadoop_client
   permitUnusedDeclared library.java.hadoop_client
-  compileOnly library.java.hadoop_common
-  compileOnly library.java.hadoop_mapreduce_client_core
+  provided library.java.hadoop_common
+  provided library.java.hadoop_mapreduce_client_core
   testImplementation library.java.junit
-  testImplementation library.java.hadoop_client
-  testImplementation library.java.hadoop_common
-  testImplementation library.java.hadoop_mapreduce_client_core
   testImplementation library.java.hamcrest
   hadoopVersions.each {kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"

sdks/java/io/hadoop-format/build.gradle

@@ -54,17 +54,25 @@ configurations.testRuntimeClasspath {
   exclude group: "org.slf4j", module: "log4j-over-slf4j"
 }
 
+// Force use the old version of JAMM that cassandra relies on
+configurations.all {
+  resolutionStrategy {
+    exclude group: "io.github.stephankoelle", module: "jamm"
+  }
+}
+
+
 dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation library.java.vendored_guava_26_0_jre
   implementation library.java.slf4j_api
   implementation project(":sdks:java:io:hadoop-common")
   implementation library.java.joda_time
-  compileOnly library.java.hadoop_common
-  compileOnly library.java.hadoop_hdfs
+  provided library.java.hadoop_common
+  provided library.java.hadoop_hdfs
   permitUnusedDeclared library.java.hadoop_hdfs
-  compileOnly library.java.hadoop_hdfs_client
-  compileOnly library.java.hadoop_mapreduce_client_core
+  provided library.java.hadoop_hdfs_client
+  provided library.java.hadoop_mapreduce_client_core
   // Ensure that the older version of JAMM that cassandra relies on appears
   // on the classpath before the one provided by :sdks:java:core shadowTest.
   testImplementation "com.github.jbellis:jamm:0.3.0"
@@ -96,9 +104,6 @@ dependencies {
   testImplementation library.java.cassandra_driver_core
   testImplementation library.java.cassandra_driver_mapping
   testImplementation "org.apache.cassandra:cassandra-all:3.11.8"
-  testImplementation library.java.hadoop_common
-  testImplementation library.java.hadoop_hdfs
-  testImplementation library.java.hadoop_mapreduce_client_core
   testImplementation library.java.postgres
   testImplementation library.java.junit
   testImplementation library.java.hamcrest

sdks/java/io/hbase/build.gradle

@@ -41,7 +41,7 @@ dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation project(":sdks:java:io:hadoop-common")
   implementation library.java.slf4j_api
-  compileOnly "org.apache.hbase:hbase-shaded-client:$hbase_version"
+  provided "org.apache.hbase:hbase-shaded-client:$hbase_version"
   testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration")
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
   testImplementation library.java.junit

sdks/java/io/hcatalog/build.gradle

@@ -77,14 +77,14 @@ dependencies {
   implementation library.java.joda_time
   // Hive bundles without repackaging Jackson which is why we redeclare it here so that it appears
   // on the compile/test/runtime classpath before Hive.
-  compileOnly library.java.jackson_annotations
-  compileOnly library.java.jackson_core
-  compileOnly library.java.jackson_databind
+  provided library.java.jackson_annotations
+  provided library.java.jackson_core
+  provided library.java.jackson_databind
   // Calcite (a dependency of Hive) bundles without repackaging Guava which is why we redeclare it
   // here so that it appears on the compile/test/runtime classpath before Calcite.
-  compileOnly library.java.hadoop_common
-  compileOnly "org.apache.hive:hive-exec:$hive_version"
-  compileOnly(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core", version: hive_version) {
+  provided library.java.hadoop_common
+  provided "org.apache.hive:hive-exec:$hive_version"
+  provided(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core", version: hive_version) {
     exclude group: "org.apache.hive", module: "hive-exec"
     exclude group: "com.google.protobuf", module: "protobuf-java"
   }
@@ -96,7 +96,6 @@ dependencies {
   testImplementation "org.apache.hive:hive-exec:$hive_version"
   testImplementation "org.apache.hive:hive-common:$hive_version"
   testImplementation "org.apache.hive:hive-cli:$hive_version"
-  testImplementation "org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version"
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   hadoopVersions.each {kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"

sdks/java/io/kafka/build.gradle

@@ -47,15 +47,15 @@ kafkaVersions.each{k,v -> configurations.create("kafkaVersion$k")}
 
 dependencies {
   implementation library.java.vendored_guava_26_0_jre
-  compileOnly library.java.jackson_dataformat_csv
+  provided library.java.jackson_dataformat_csv
   permitUnusedDeclared library.java.jackson_dataformat_csv
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation project(":runners:core-construction-java")
   implementation project(":sdks:java:expansion-service")
   permitUnusedDeclared project(":sdks:java:expansion-service") // BEAM-11761
   implementation library.java.avro
-  // Get back to "compileOnly" since 2.14
-  compileOnly library.java.kafka_clients
+  // Get back to "provided" since 2.14
+  provided library.java.kafka_clients
   testImplementation library.java.kafka_clients
   implementation library.java.slf4j_api
   implementation library.java.joda_time
@@ -66,14 +66,14 @@ dependencies {
     // zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current
     // "spotbugs-annotations:3.1.12" used in Beam. Not required.
     exclude group: "org.apache.zookeeper", module: "zookeeper"
-    // "kafka-clients" has to be compileOnly since user can use its own version.
+    // "kafka-clients" has to be provided since user can use its own version.
     exclude group: "org.apache.kafka", module: "kafka-clients"
   }
   implementation ("io.confluent:kafka-schema-registry-client:5.3.2") {
     // It depends on "spotbugs-annotations:3.1.9" which clashes with current
     // "spotbugs-annotations:3.1.12" used in Beam. Not required.
     exclude group: "org.apache.zookeeper", module: "zookeeper"
-    // "kafka-clients" has to be compileOnly since user can use its own version.
+    // "kafka-clients" has to be provided since user can use its own version.
     exclude group: "org.apache.kafka", module: "kafka-clients"
   }
   testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")