From f9741af86434793c8a9ab145d6593d435ba163d7 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 10:16:39 -0500 Subject: [PATCH 01/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- buildSrc/build.gradle.kts | 1 - .../beam/gradle/BeamModulePlugin.groovy | 6 +++--- examples/java/build.gradle | 4 ++-- examples/java/twitter/build.gradle | 4 ++-- examples/kotlin/build.gradle | 2 +- runners/direct-java/build.gradle | 4 ++-- runners/flink/flink_runner.gradle | 2 +- .../worker/legacy-worker/build.gradle | 8 ++++---- runners/java-fn-execution/build.gradle | 2 +- .../spark/job-server/spark_job_server.gradle | 4 ++-- runners/spark/spark_runner.gradle | 20 +++++++++---------- sdks/java/container/build.gradle | 2 +- sdks/java/core/build.gradle | 10 +++++----- .../google-cloud-platform-core/build.gradle | 4 ++-- sdks/java/extensions/ml/build.gradle | 2 +- sdks/java/extensions/sorter/build.gradle | 4 ++-- sdks/java/extensions/sql/build.gradle | 14 ++++++------- .../extensions/sql/datacatalog/build.gradle | 6 +++--- .../java/extensions/sql/hcatalog/build.gradle | 4 ++-- sdks/java/extensions/sql/zetasql/build.gradle | 2 +- sdks/java/fn-execution/build.gradle | 2 +- sdks/java/io/debezium/build.gradle | 2 +- sdks/java/io/hadoop-common/build.gradle | 6 +++--- sdks/java/io/hadoop-file-system/build.gradle | 6 +++--- sdks/java/io/hadoop-format/build.gradle | 6 +++--- sdks/java/io/hbase/build.gradle | 2 +- sdks/java/io/hcatalog/build.gradle | 12 +++++------ sdks/java/io/kafka/build.gradle | 10 +++++----- sdks/java/io/parquet/build.gradle | 2 +- sdks/java/io/rabbitmq/build.gradle | 2 +- sdks/java/testing/tpcds/build.gradle | 2 +- 31 files changed, 78 insertions(+), 79 deletions(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 47df89e22d4f..3339eba5ddc3 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -43,7 +43,6 @@ dependencies { runtimeOnly("net.ltgt.gradle:gradle-apt-plugin:0.21") // Enable a Java annotation processor runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation - runtimeOnly("io.spring.gradle:propdeps-plugin:0.0.9.RELEASE") // Enable provided and optional configurations runtimeOnly("com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0") // Enable Avro code generation runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin runtimeOnly("gradle.plugin.com.github.blindpirate:gogradle:0.11.4") // Enable Go code compilation diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 469afc2913dd..661211502dcf 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1254,7 +1254,7 @@ class BeamModulePlugin implements Plugin { project.dependencies { shadowTestRuntimeClasspath it.project(path: project.path, configuration: "shadowTest") - shadowTestRuntimeClasspath it.project(path: project.path, configuration: "provided") + shadowTestRuntimeClasspath it.project(path: project.path, configuration: "compileOnly") } project.test { classpath = project.configurations.shadowTestRuntimeClasspath } @@ -1580,8 +1580,8 @@ class BeamModulePlugin implements Plugin { // TODO: Should we use the runtime scope instead of the compile scope // which forces all our consumers to declare what they consume? generateDependenciesFromConfiguration( - configuration: (configuration.shadowClosure ? 'shadow' : 'compile'), scope: 'compile') - generateDependenciesFromConfiguration(configuration: 'provided', scope: 'provided') + configuration: (configuration.shadowClosure ? 'shadow' : 'implementation'), scope: 'compile') + generateDependenciesFromConfiguration(configuration: 'compileOnly', scope: 'provided') if (!boms.isEmpty()) { def dependencyManagementNode = root.appendNode('dependencyManagement') diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 677317d4fd65..dba5320f02f2 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -77,8 +77,8 @@ dependencies { implementation library.java.proto_google_cloud_bigtable_v2 implementation library.java.proto_google_cloud_datastore_v1 implementation library.java.slf4j_api - provided library.java.commons_io - provided library.java.commons_csv + implementation library.java.commons_io + implementation library.java.commons_csv runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") implementation library.java.vendored_grpc_1_36_0 implementation library.java.vendored_guava_26_0_jre diff --git a/examples/java/twitter/build.gradle b/examples/java/twitter/build.gradle index f707ae201d0b..693e6ef9503f 100644 --- a/examples/java/twitter/build.gradle +++ b/examples/java/twitter/build.gradle @@ -50,8 +50,8 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.joda_time implementation library.java.slf4j_api - provided library.java.commons_io - provided library.java.commons_csv + implementation library.java.commons_io + implementation library.java.commons_csv runtimeOnly project(path: ":runners:direct-java", configuration: "shadow") implementation ("org.twitter4j:twitter4j-stream:4.0.7") implementation ("org.twitter4j:twitter4j-core:4.0.7") diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 1c56e3fc2d4b..3e78155230ed 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -52,7 +52,7 @@ dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation library.java.vendored_guava_26_0_jre implementation project(path: ":sdks:java:core", configuration: "shadow") - // Add the dependency that sdks:java:core that is marked as provided + // Add the dependency that sdks:java:core that is marked as compileOnly implementation library.java.hamcrest_core implementation project(":sdks:java:extensions:google-cloud-platform-core") implementation project(":sdks:java:io:google-cloud-platform") diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index 0fab169596b4..6219c217e13d 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -74,8 +74,8 @@ dependencies { shadow library.java.joda_time shadow library.java.slf4j_api shadow library.java.jackson_databind - provided library.java.hamcrest_core - provided library.java.junit + compileOnly library.java.hamcrest_core + compileOnly library.java.junit shadowTest project(path: ":sdks:java:core", configuration: "shadowTest") shadowTest project(path: ":runners:core-java", configuration: "testRuntimeMigration") shadowTest library.java.mockito_core diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index 49d22f88f785..fb6c150d7c97 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -148,7 +148,7 @@ dependencies { implementation "org.apache.flink:flink-runtime_2.11:$flink_version" implementation "org.apache.flink:flink-streaming-java_2.11:$flink_version" // RocksDB state backend (included in the Flink distribution) - provided "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version" + compileOnly "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version" testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") // FlinkStateInternalsTest extends abstract StateInternalsTest testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") diff --git a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle index a0886dddcb02..8e8263d1e30f 100644 --- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle @@ -189,13 +189,13 @@ dependencies { // runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided // there. sdk_provided_dependencies.each { - provided(it) + compileOnly(it) } sdk_provided_shaded_project_dependencies.each { - provided project(path: it, configuration: "shadow") + compileOnly project(path: it, configuration: "shadow") } sdk_provided_project_dependencies.each { - provided project(it) + compileOnly project(it) } implementation project(path: ":model:fn-execution", configuration: "shadow") @@ -223,7 +223,7 @@ dependencies { implementation library.java.joda_time implementation library.java.slf4j_api implementation library.java.vendored_grpc_1_36_0 - provided library.java.error_prone_annotations + implementation library.java.error_prone_annotations runtimeOnly library.java.slf4j_jdk14 // Any test dependency which intersects with our relocation rules above needs to be relocated diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index 4b42fecd0d97..4c8c86c510ed 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -25,7 +25,7 @@ description = "Apache Beam :: Runners :: Java Fn Execution" dependencies { implementation library.java.vendored_guava_26_0_jre implementation project(":runners:core-construction-java") - provided project(":sdks:java:harness") + compileOnly project(":sdks:java:harness") implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(path: ":model:fn-execution", configuration: "shadow") implementation project(path: ":sdks:java:core", configuration: "shadow") diff --git a/runners/spark/job-server/spark_job_server.gradle b/runners/spark/job-server/spark_job_server.gradle index 7d0acf6588b7..09590941e3cd 100644 --- a/runners/spark/job-server/spark_job_server.gradle +++ b/runners/spark/job-server/spark_job_server.gradle @@ -52,9 +52,9 @@ configurations.all { dependencies { implementation project(sparkRunnerProject) permitUnusedDeclared project(sparkRunnerProject) - implementation project(path: sparkRunnerProject, configuration: "provided") + implementation project(path: sparkRunnerProject, configuration: "compileOnly") validatesPortableRunner project(path: sparkRunnerProject, configuration: "testRuntimeMigration") - validatesPortableRunner project(path: sparkRunnerProject, configuration: "provided") + validatesPortableRunner project(path: sparkRunnerProject, configuration: "compileOnly") validatesPortableRunner project(path: ":sdks:java:core", configuration: "shadowTest") validatesPortableRunner project(path: ":runners:core-java", configuration: "testRuntimeMigration") validatesPortableRunner project(path: ":runners:portability:java", configuration: "testRuntimeMigration") diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 13307cb8715c..faa2cbbda525 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -162,10 +162,10 @@ dependencies { implementation project(":sdks:java:fn-execution") implementation library.java.vendored_grpc_1_36_0 implementation library.java.vendored_guava_26_0_jre - provided "org.apache.spark:spark-core_$spark_scala_version:$spark_version" - provided "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version" - provided "org.apache.spark:spark-sql_$spark_scala_version:$spark_version" - provided "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version" + compileOnly "org.apache.spark:spark-core_$spark_scala_version:$spark_version" + compileOnly "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version" + compileOnly "org.apache.spark:spark-sql_$spark_scala_version:$spark_version" + compileOnly "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version" if(project.property("spark_scala_version").equals("2.11")){ runtimeOnly library.java.jackson_module_scala_2_11 } else { @@ -173,11 +173,11 @@ dependencies { } // Force paranamer 2.8 to avoid issues when using Scala 2.12 runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8" - provided library.java.hadoop_common - provided library.java.commons_io - provided library.java.hamcrest_core - provided library.java.hamcrest_library - provided "com.esotericsoftware.kryo:kryo:2.21" + compileOnly library.java.hadoop_common + compileOnly library.java.commons_io + compileOnly library.java.hamcrest_core + compileOnly library.java.hamcrest_library + compileOnly "com.esotericsoftware.kryo:kryo:2.21" testImplementation project(":sdks:java:io:kafka") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") // SparkStateInternalsTest extends abstract StateInternalsTest @@ -196,7 +196,7 @@ dependencies { validatesRunner project(path: ":examples:java", configuration: "testRuntimeMigration") validatesRunner project(path: project.path, configuration: "testRuntimeMigration") validatesRunner project(project.path) - validatesRunner project(path: project.path, configuration: "provided") + validatesRunner project(path: project.path, configuration: "compileOnly") hadoopVersions.each {kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value" } diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle index eae896cf91b5..142e4aaefe08 100644 --- a/sdks/java/container/build.gradle +++ b/sdks/java/container/build.gradle @@ -40,7 +40,7 @@ dependencies { dockerDependency project(path: ":sdks:java:harness", configuration: "shadow") // For executing KafkaIO, e.g. as an external transform dockerDependency project(":sdks:java:io:kafka") - // This dependency is set to 'provided' scope in :sdks:java:io:kafka + // This dependency is set to 'compileOnly' scope in :sdks:java:io:kafka dockerDependency library.java.kafka_clients } diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index 6f1df2b42791..deec8cbcd438 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -90,11 +90,11 @@ dependencies { shadow library.java.avro shadow library.java.snappy_java shadow library.java.joda_time - provided library.java.junit - provided library.java.hamcrest_core - provided library.java.hamcrest_library - provided 'io.airlift:aircompressor:0.18' - provided 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1' + compileOnly library.java.junit + compileOnly library.java.hamcrest_core + compileOnly library.java.hamcrest_library + compileOnly 'io.airlift:aircompressor:0.18' + compileOnly 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1' shadowTest library.java.jackson_dataformat_yaml shadowTest library.java.guava_testlib shadowTest library.java.mockito_core diff --git a/sdks/java/extensions/google-cloud-platform-core/build.gradle b/sdks/java/extensions/google-cloud-platform-core/build.gradle index 90c84d98c268..4aa1c98faa0e 100644 --- a/sdks/java/extensions/google-cloud-platform-core/build.gradle +++ b/sdks/java/extensions/google-cloud-platform-core/build.gradle @@ -55,8 +55,8 @@ dependencies { implementation library.java.jackson_annotations implementation library.java.jackson_databind permitUnusedDeclared library.java.jackson_databind // BEAM-11761 - provided library.java.hamcrest_core - provided library.java.junit + compileOnly library.java.hamcrest_core + compileOnly library.java.junit testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.mockito_core testRuntimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/extensions/ml/build.gradle b/sdks/java/extensions/ml/build.gradle index 6f001427993c..4d84986a7998 100644 --- a/sdks/java/extensions/ml/build.gradle +++ b/sdks/java/extensions/ml/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation library.java.gax implementation library.java.protobuf_java implementation library.java.slf4j_api - provided library.java.junit + compileOnly library.java.junit testImplementation project(path: ':sdks:java:core', configuration: 'shadowTest') implementation 'com.google.cloud:google-cloud-vision:1.99.3' testImplementation library.java.mockito_core diff --git a/sdks/java/extensions/sorter/build.gradle b/sdks/java/extensions/sorter/build.gradle index 2374ac97d232..3ce3a66c84ab 100644 --- a/sdks/java/extensions/sorter/build.gradle +++ b/sdks/java/extensions/sorter/build.gradle @@ -36,8 +36,8 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.vendored_guava_26_0_jre implementation library.java.slf4j_api - provided library.java.hadoop_mapreduce_client_core - provided library.java.hadoop_common + compileOnly library.java.hadoop_mapreduce_client_core + compileOnly library.java.hadoop_common testImplementation library.java.mockito_core testImplementation library.java.junit testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index c722f3c4f9f7..2a909ea1bbf6 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -54,7 +54,7 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} dependencies { // TODO(BEAM-13179): Determine how to build without this dependency - provided "org.immutables:value:2.8.8" + compileOnly "org.immutables:value:2.8.8" javacc "net.java.dev.javacc:javacc:4.0" fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0" fmppTask "org.freemarker:freemarker:2.3.31" @@ -77,14 +77,14 @@ dependencies { implementation library.java.slf4j_api implementation library.java.joda_time implementation library.java.vendored_guava_26_0_jre - provided project(":sdks:java:io:kafka") - provided project(":sdks:java:io:google-cloud-platform") + compileOnly project(":sdks:java:io:kafka") + compileOnly project(":sdks:java:io:google-cloud-platform") implementation project(":sdks:java:io:mongodb") implementation library.java.avro - provided project(":sdks:java:io:parquet") - provided library.java.jackson_dataformat_xml - provided library.java.hadoop_client - provided library.java.kafka_clients + compileOnly project(":sdks:java:io:parquet") + compileOnly library.java.jackson_dataformat_xml + compileOnly library.java.hadoop_client + compileOnly library.java.kafka_clients testImplementation library.java.vendored_calcite_1_28_0 testImplementation library.java.vendored_guava_26_0_jre testImplementation library.java.junit diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 55f42b8461dc..9460bdb91ff0 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -36,11 +36,11 @@ dependencies { implementation library.java.vendored_guava_26_0_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.threeten:threetenbp:1.4.5" - provided project(":sdks:java:extensions:sql") + compileOnly project(":sdks:java:extensions:sql") // Dependencies for the example - provided project(":sdks:java:io:google-cloud-platform") - provided library.java.slf4j_api + compileOnly project(":sdks:java:io:google-cloud-platform") + compileOnly library.java.slf4j_api testImplementation project(":sdks:java:extensions:sql:zetasql") testRuntimeOnly library.java.slf4j_simple diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index dd052ee27982..053f83af9d4e 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -43,8 +43,8 @@ configurations.all { } dependencies { - provided project(":sdks:java:extensions:sql") - provided project(":sdks:java:io:hcatalog") + compileOnly project(":sdks:java:extensions:sql") + compileOnly project(":sdks:java:io:hcatalog") // Needed for HCatalogTableProvider tests, // they use HCat* types diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 739fc8b28505..d6a84becf230 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -31,7 +31,7 @@ def zetasql_version = "2020.10.1" dependencies { // TODO(BEAM-13179): Determine how to build without this dependency - provided "org.immutables:value:2.8.8" + compileOnly "org.immutables:value:2.8.8" implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":sdks:java:core", configuration: "shadow") diff --git a/sdks/java/fn-execution/build.gradle b/sdks/java/fn-execution/build.gradle index 5e3faf6a813b..e7ad5ec74187 100644 --- a/sdks/java/fn-execution/build.gradle +++ b/sdks/java/fn-execution/build.gradle @@ -33,7 +33,7 @@ dependencies { implementation library.java.vendored_guava_26_0_jre implementation library.java.slf4j_api implementation library.java.joda_time - provided library.java.junit + compileOnly library.java.junit testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit testImplementation library.java.mockito_core diff --git a/sdks/java/io/debezium/build.gradle b/sdks/java/io/debezium/build.gradle index 565541755d74..7f08371ea094 100644 --- a/sdks/java/io/debezium/build.gradle +++ b/sdks/java/io/debezium/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.slf4j_api implementation library.java.joda_time - provided library.java.jackson_dataformat_csv + compileOnly library.java.jackson_dataformat_csv testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") diff --git a/sdks/java/io/hadoop-common/build.gradle b/sdks/java/io/hadoop-common/build.gradle index 4cf57dc437d6..a6af7b7ea218 100644 --- a/sdks/java/io/hadoop-common/build.gradle +++ b/sdks/java/io/hadoop-common/build.gradle @@ -35,9 +35,9 @@ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") - provided library.java.hadoop_client - provided library.java.hadoop_common - provided library.java.hadoop_mapreduce_client_core + compileOnly library.java.hadoop_client + compileOnly library.java.hadoop_common + compileOnly library.java.hadoop_mapreduce_client_core testImplementation library.java.junit hadoopVersions.each {kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value" diff --git a/sdks/java/io/hadoop-file-system/build.gradle b/sdks/java/io/hadoop-file-system/build.gradle index cbc8ea3c7c6a..db1679df2073 100644 --- a/sdks/java/io/hadoop-file-system/build.gradle +++ b/sdks/java/io/hadoop-file-system/build.gradle @@ -40,9 +40,9 @@ dependencies { implementation library.java.jackson_core implementation library.java.jackson_databind implementation library.java.slf4j_api - provided library.java.hadoop_client - provided library.java.hadoop_common - provided library.java.hadoop_mapreduce_client_core + compileOnly library.java.hadoop_client + compileOnly library.java.hadoop_common + compileOnly library.java.hadoop_mapreduce_client_core testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.mockito_core testImplementation library.java.junit diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle index 054362d0cec4..e616b91aa5db 100644 --- a/sdks/java/io/hadoop-format/build.gradle +++ b/sdks/java/io/hadoop-format/build.gradle @@ -60,9 +60,9 @@ dependencies { implementation library.java.slf4j_api implementation project(":sdks:java:io:hadoop-common") implementation library.java.joda_time - provided library.java.hadoop_common - provided library.java.hadoop_hdfs - provided library.java.hadoop_mapreduce_client_core + compileOnly library.java.hadoop_common + compileOnly library.java.hadoop_hdfs + compileOnly library.java.hadoop_mapreduce_client_core testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") testImplementation project(":sdks:java:testing:test-utils") diff --git a/sdks/java/io/hbase/build.gradle b/sdks/java/io/hbase/build.gradle index d40d25098c0f..92b9a4062e32 100644 --- a/sdks/java/io/hbase/build.gradle +++ b/sdks/java/io/hbase/build.gradle @@ -41,7 +41,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(":sdks:java:io:hadoop-common") implementation library.java.slf4j_api - provided "org.apache.hbase:hbase-shaded-client:$hbase_version" + compileOnly "org.apache.hbase:hbase-shaded-client:$hbase_version" testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index d1e8eabb5a3c..7e23ce8b0d6b 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -72,14 +72,14 @@ dependencies { implementation library.java.joda_time // Hive bundles without repackaging Jackson which is why we redeclare it here so that it appears // on the compile/test/runtime classpath before Hive. - provided library.java.jackson_annotations - provided library.java.jackson_core - provided library.java.jackson_databind + compileOnly library.java.jackson_annotations + compileOnly library.java.jackson_core + compileOnly library.java.jackson_databind // Calcite (a dependency of Hive) bundles without repackaging Guava which is why we redeclare it // here so that it appears on the compile/test/runtime classpath before Calcite. - provided library.java.hadoop_common - provided "org.apache.hive:hive-exec:$hive_version" - provided(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core", version: hive_version) { + compileOnly library.java.hadoop_common + compileOnly "org.apache.hive:hive-exec:$hive_version" + compileOnly(group: "org.apache.hive.hcatalog", name: "hive-hcatalog-core", version: hive_version) { exclude group: "org.apache.hive", module: "hive-exec" exclude group: "com.google.protobuf", module: "protobuf-java" } diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index b361980240fa..35860b90f8a5 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -52,8 +52,8 @@ dependencies { implementation project(":sdks:java:expansion-service") permitUnusedDeclared project(":sdks:java:expansion-service") // BEAM-11761 implementation library.java.avro - // Get back to "provided" since 2.14 - provided library.java.kafka_clients + // Get back to "compileOnly" since 2.14 + compileOnly library.java.kafka_clients implementation library.java.slf4j_api implementation library.java.joda_time implementation library.java.jackson_annotations @@ -63,17 +63,17 @@ dependencies { // zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current // "spotbugs-annotations:3.1.12" used in Beam. Not required. exclude group: "org.apache.zookeeper", module: "zookeeper" - // "kafka-clients" has to be provided since user can use its own version. + // "kafka-clients" has to be compileOnly since user can use its own version. exclude group: "org.apache.kafka", module: "kafka-clients" } implementation ("io.confluent:kafka-schema-registry-client:5.3.2") { // It depends on "spotbugs-annotations:3.1.9" which clashes with current // "spotbugs-annotations:3.1.12" used in Beam. Not required. exclude group: "org.apache.zookeeper", module: "zookeeper" - // "kafka-clients" has to be provided since user can use its own version. + // "kafka-clients" has to be compileOnly since user can use its own version. exclude group: "org.apache.kafka", module: "kafka-clients" } - provided library.java.jackson_dataformat_csv + compileOnly library.java.jackson_dataformat_csv testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation project(":sdks:java:io:synthetic") testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") diff --git a/sdks/java/io/parquet/build.gradle b/sdks/java/io/parquet/build.gradle index b9fc223dad0c..a755dec51203 100644 --- a/sdks/java/io/parquet/build.gradle +++ b/sdks/java/io/parquet/build.gradle @@ -47,7 +47,7 @@ dependencies { implementation "org.apache.parquet:parquet-common:$parquet_version" implementation "org.apache.parquet:parquet-hadoop:$parquet_version" implementation library.java.avro - provided library.java.hadoop_client + compileOnly library.java.hadoop_client testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit testRuntimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/io/rabbitmq/build.gradle b/sdks/java/io/rabbitmq/build.gradle index e9392c35be48..49d516e5fed4 100644 --- a/sdks/java/io/rabbitmq/build.gradle +++ b/sdks/java/io/rabbitmq/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation project(path: ":sdks:java:core", configuration: "shadow") implementation library.java.joda_time implementation "com.rabbitmq:amqp-client:5.7.3" - provided library.java.jackson_dataformat_xml + compileOnly library.java.jackson_dataformat_xml testImplementation project(path: ":sdks:java:io:common", configuration: "testRuntimeMigration") testImplementation "org.apache.qpid:qpid-broker-core:$qpid_version" testImplementation "org.apache.qpid:qpid-broker-plugins-memory-store:$qpid_version" diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 6e4f27ad74b9..d0eeb9dea2c6 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -57,7 +57,7 @@ dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) // TODO(BEAM-13179): Determine how to build without this dependency - provided "org.immutables:value:2.8.8" + compileOnly "org.immutables:value:2.8.8" implementation library.java.avro implementation library.java.vendored_guava_26_0_jre implementation library.java.vendored_calcite_1_28_0 From 6d4f4cf4b8a27b4f71e406317e2586c1022070ed Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 10:25:33 -0500 Subject: [PATCH 02/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 ---------- ownership/JAVA_DEPENDENCY_OWNERS.yaml | 5 ----- 2 files changed, 15 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 661211502dcf..a91d62e1ac5e 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -748,9 +748,6 @@ class BeamModulePlugin implements Plugin { // * java // * maven // * net.ltgt.apt (plugin to configure annotation processing tool) - // * propdeps (provide optional and provided dependency configurations) - // * propdeps-maven - // * propdeps-idea // * checkstyle // * spotbugs // * shadow (conditional on shadowClosure being specified) @@ -1032,13 +1029,6 @@ class BeamModulePlugin implements Plugin { implementation "org.checkerframework:checker-qual:$checkerframework_version" } - // Add the optional and provided configurations for dependencies - // TODO: Either remove these plugins and find another way to generate the Maven poms - // with the correct dependency scopes configured. - project.apply plugin: 'propdeps' - project.apply plugin: 'propdeps-maven' - project.apply plugin: 'propdeps-idea' - // Defines Targets for sonarqube analysis reporting. project.apply plugin: "org.sonarqube" diff --git a/ownership/JAVA_DEPENDENCY_OWNERS.yaml b/ownership/JAVA_DEPENDENCY_OWNERS.yaml index 904f4404660e..ad117f2eea65 100644 --- a/ownership/JAVA_DEPENDENCY_OWNERS.yaml +++ b/ownership/JAVA_DEPENDENCY_OWNERS.yaml @@ -514,11 +514,6 @@ deps: artifact: opencensus-contrib-grpc-metrics owners: - io.spring.gradle:propdeps-plugin: - group: io.spring.gradle - artifact: propdeps-plugin - owners: - javax.xml.bind:jaxb-api: group: javax.xml.bind artifact: jaxb-api From 81c56ac45b10545f77ef13db7158f6c87fc04e27 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 10:31:59 -0500 Subject: [PATCH 03/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- runners/java-fn-execution/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index 4c8c86c510ed..45e0d4c19b41 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -25,6 +25,7 @@ description = "Apache Beam :: Runners :: Java Fn Execution" dependencies { implementation library.java.vendored_guava_26_0_jre implementation project(":runners:core-construction-java") + implementation project(":runners:core-java") compileOnly project(":sdks:java:harness") implementation project(path: ":model:pipeline", configuration: "shadow") implementation project(path: ":model:fn-execution", configuration: "shadow") @@ -37,7 +38,6 @@ dependencies { implementation library.java.joda_time testImplementation project(":sdks:java:harness") testImplementation project(":runners:core-construction-java") - testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit testImplementation library.java.mockito_core From 011e4896ee66f5148e734f42963c69f3ea18f585 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 11:08:21 -0500 Subject: [PATCH 04/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- .../worker/legacy-worker/build.gradle | 7 ++++++- .../beam/runners/dataflow/worker/status/DebugCapture.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle index 8e8263d1e30f..6e9ecf1e3b62 100644 --- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle @@ -185,6 +185,8 @@ compileTestJava { } dependencies { + implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) + // Note that any dependency that is modified here should also be modified within // runners/google-cloud-dataflow-java/worker/build.gradle using the rules provided // there. @@ -195,7 +197,7 @@ dependencies { compileOnly project(path: it, configuration: "shadow") } sdk_provided_project_dependencies.each { - compileOnly project(it) + compileOnly project(path: it) } implementation project(path: ":model:fn-execution", configuration: "shadow") @@ -208,6 +210,9 @@ dependencies { implementation project(path: ":runners:google-cloud-dataflow-java:worker:windmill", configuration: "shadow") shadow library.java.vendored_guava_26_0_jre + implementation library.java.google_auth_library_credentials + implementation library.java.proto_google_common_protos + // Conscrypt shouldn't be included here because Conscrypt won't work when being shaded. // (Context: https://github.com/apache/beam/pull/13846) // Conscrypt will be added to runtime dependencies by GrpcVendoring so compileOnly works for now. diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java index 21691b106489..b65410b3cc63 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/status/DebugCapture.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.json.gson.GsonFactory; import com.google.api.client.util.Key; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.GetDebugConfigRequest; @@ -50,7 +50,7 @@ }) public class DebugCapture { private static final Logger LOG = LoggerFactory.getLogger(DebugCapture.class); - private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + private static final JsonFactory JSON_FACTORY = new GsonFactory(); private static final String COMPONENT = "JavaHarness"; // How often to refresh debug capture config. private static final long UPDATE_CONFIG_PERIOD_SEC = 60; From 4de55c3dcfbf461702458474d703b888b5030247 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 11:25:59 -0500 Subject: [PATCH 05/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- sdks/java/extensions/sql/build.gradle | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 2a909ea1bbf6..6f1cb10093b0 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -53,6 +53,8 @@ def hadoopVersions = [ hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} dependencies { + implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) + // TODO(BEAM-13179): Determine how to build without this dependency compileOnly "org.immutables:value:2.8.8" javacc "net.java.dev.javacc:javacc:4.0" @@ -78,7 +80,12 @@ dependencies { implementation library.java.joda_time implementation library.java.vendored_guava_26_0_jre compileOnly project(":sdks:java:io:kafka") - compileOnly project(":sdks:java:io:google-cloud-platform") + implementation project(":sdks:java:extensions:google-cloud-platform-core") + implementation project(":sdks:java:io:google-cloud-platform") + implementation library.java.proto_google_cloud_bigtable_v2 + implementation library.java.google_api_services_bigquery + implementation library.java.proto_google_cloud_pubsublite_v1 + implementation library.java.google_cloud_pubsublite implementation project(":sdks:java:io:mongodb") implementation library.java.avro compileOnly project(":sdks:java:io:parquet") From e5c46d1197f6ea75ca33567dbbe7f5bf07dbd522 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 11:33:20 -0500 Subject: [PATCH 06/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- sdks/java/core/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/java/core/build.gradle b/sdks/java/core/build.gradle index deec8cbcd438..fa5179457fb9 100644 --- a/sdks/java/core/build.gradle +++ b/sdks/java/core/build.gradle @@ -105,4 +105,6 @@ dependencies { shadowTest library.java.avro_tests shadowTest library.java.zstd_jni testRuntimeOnly library.java.slf4j_jdk14 + testImplementation 'io.airlift:aircompressor:0.18' + testImplementation 'com.facebook.presto.hadoop:hadoop-apache2:3.2.0-1' } From 8e10e0cf44862a2f688fd7e1866a21f5b288cd59 Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 12:38:33 -0500 Subject: [PATCH 07/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- sdks/java/extensions/sql/build.gradle | 9 ++++++++- sdks/java/extensions/sql/datacatalog/build.gradle | 8 ++++++++ sdks/java/extensions/sql/jdbc/build.gradle | 7 +------ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 6f1cb10093b0..b7510931d2da 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -97,9 +97,16 @@ dependencies { testImplementation library.java.junit testImplementation library.java.quickcheck_core testImplementation library.java.testcontainers_kafka + testImplementation library.java.google_cloud_bigtable_client_core testImplementation library.java.google_cloud_bigtable_emulator + testImplementation library.java.proto_google_cloud_bigtable_admin_v2 + testImplementation library.java.proto_google_cloud_datastore_v1 + testImplementation library.java.google_cloud_datastore_v1_proto_client + testImplementation library.java.hadoop_client + testImplementation library.java.kafka_clients + testImplementation project(":sdks:java:io:kafka") + testImplementation project(":sdks:java:io:parquet") testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration") - testImplementation project(":sdks:java:io:thrift") testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider") diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 9460bdb91ff0..78dd98e07818 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -34,15 +34,23 @@ dependencies { implementation library.java.protobuf_java implementation library.java.slf4j_api implementation library.java.vendored_guava_26_0_jre + implementation library.java.vendored_calcite_1_28_0 + implementation "com.alibaba:fastjson:1.2.69" implementation project(path: ":sdks:java:core", configuration: "shadow") implementation "org.threeten:threetenbp:1.4.5" compileOnly project(":sdks:java:extensions:sql") // Dependencies for the example + implementation project(":sdks:java:extensions:google-cloud-platform-core") compileOnly project(":sdks:java:io:google-cloud-platform") compileOnly library.java.slf4j_api + testImplementation project(":sdks:java:extensions:sql") testImplementation project(":sdks:java:extensions:sql:zetasql") + testImplementation project(":runners:direct-java") + testImplementation project(":sdks:java:io:google-cloud-platform") + testImplementation library.java.google_api_services_bigquery + testImplementation library.java.junit testRuntimeOnly library.java.slf4j_simple } diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 1e376b75f505..7d3158c3da1d 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -60,12 +60,7 @@ shadowJar { } } -test { - doFirst { - // Assert everything is in the output or test jar - assert classpath.size() == 2 - } -} +test {} task shadowJarTest(type: Test, dependsOn: ":sdks:java:extensions:sql:jdbc:shadowJar") { group = "Verification" From 9620cdec99cedec3c7c85eafa329e253562ee5ae Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 13:53:13 -0500 Subject: [PATCH 08/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- runners/flink/flink_runner.gradle | 1 + .../worker/legacy-worker/build.gradle | 3 +++ runners/java-fn-execution/build.gradle | 1 + runners/spark/spark_runner.gradle | 9 +++++++++ sdks/java/extensions/sorter/build.gradle | 2 ++ sdks/java/extensions/sql/build.gradle | 2 ++ sdks/java/extensions/sql/hcatalog/build.gradle | 7 +++++-- sdks/java/io/hadoop-common/build.gradle | 3 +++ sdks/java/io/hadoop-format/build.gradle | 3 +++ sdks/java/io/hcatalog/build.gradle | 1 + sdks/java/io/kafka/build.gradle | 1 + sdks/java/io/parquet/build.gradle | 1 + 12 files changed, 32 insertions(+), 2 deletions(-) diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index fb6c150d7c97..19204a58577d 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -149,6 +149,7 @@ dependencies { implementation "org.apache.flink:flink-streaming-java_2.11:$flink_version" // RocksDB state backend (included in the Flink distribution) compileOnly "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version" + testImplementation "org.apache.flink:flink-statebackend-rocksdb_2.11:$flink_version" testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") // FlinkStateInternalsTest extends abstract StateInternalsTest testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") diff --git a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle index 6e9ecf1e3b62..316a8efe6200 100644 --- a/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/legacy-worker/build.gradle @@ -192,12 +192,15 @@ dependencies { // there. sdk_provided_dependencies.each { compileOnly(it) + testImplementation(it) } sdk_provided_shaded_project_dependencies.each { compileOnly project(path: it, configuration: "shadow") + testImplementation project(path: it, configuration: "shadow") } sdk_provided_project_dependencies.each { compileOnly project(path: it) + testImplementation project(path: it) } implementation project(path: ":model:fn-execution", configuration: "shadow") diff --git a/runners/java-fn-execution/build.gradle b/runners/java-fn-execution/build.gradle index 45e0d4c19b41..bbdba0b79db2 100644 --- a/runners/java-fn-execution/build.gradle +++ b/runners/java-fn-execution/build.gradle @@ -39,6 +39,7 @@ dependencies { testImplementation project(":sdks:java:harness") testImplementation project(":runners:core-construction-java") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation project(path: ":runners:core-java", configuration: "testRuntimeMigration") testImplementation library.java.junit testImplementation library.java.mockito_core testRuntimeOnly library.java.slf4j_simple diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index faa2cbbda525..1c70a59e43d7 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -178,6 +178,15 @@ dependencies { compileOnly library.java.hamcrest_core compileOnly library.java.hamcrest_library compileOnly "com.esotericsoftware.kryo:kryo:2.21" + testImplementation "org.apache.spark:spark-core_$spark_scala_version:$spark_version" + testImplementation "org.apache.spark:spark-network-common_$spark_scala_version:$spark_version" + testImplementation "org.apache.spark:spark-sql_$spark_scala_version:$spark_version" + testImplementation "org.apache.spark:spark-streaming_$spark_scala_version:$spark_version" + testImplementation library.java.hadoop_common + testImplementation library.java.commons_io + testImplementation library.java.hamcrest_core + testImplementation library.java.hamcrest_library + testImplementation "com.esotericsoftware.kryo:kryo:2.21" testImplementation project(":sdks:java:io:kafka") testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") // SparkStateInternalsTest extends abstract StateInternalsTest diff --git a/sdks/java/extensions/sorter/build.gradle b/sdks/java/extensions/sorter/build.gradle index 3ce3a66c84ab..50093ae53655 100644 --- a/sdks/java/extensions/sorter/build.gradle +++ b/sdks/java/extensions/sorter/build.gradle @@ -40,6 +40,8 @@ dependencies { compileOnly library.java.hadoop_common testImplementation library.java.mockito_core testImplementation library.java.junit + testImplementation library.java.hadoop_mapreduce_client_core + testImplementation library.java.hadoop_common testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") hadoopVersions.each{kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value" diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index b7510931d2da..5cce99701b5a 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -107,7 +107,9 @@ dependencies { testImplementation project(":sdks:java:io:kafka") testImplementation project(":sdks:java:io:parquet") testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration") + testImplementation project(path: ":sdks:java:io:thrift") testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration") + testImplementation project(":sdks:java:extensions:protobuf") testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider") testRuntimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 053f83af9d4e..cb6380f0f2c8 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -43,8 +43,11 @@ configurations.all { } dependencies { - compileOnly project(":sdks:java:extensions:sql") - compileOnly project(":sdks:java:io:hcatalog") + implementation project(":sdks:java:extensions:sql") + implementation project(":sdks:java:io:hcatalog") + implementation project(":sdks:java:core") + implementation "com.alibaba:fastjson:1.2.69" + implementation library.java.vendored_guava_26_0_jre // Needed for HCatalogTableProvider tests, // they use HCat* types diff --git a/sdks/java/io/hadoop-common/build.gradle b/sdks/java/io/hadoop-common/build.gradle index a6af7b7ea218..a742bc99ed3b 100644 --- a/sdks/java/io/hadoop-common/build.gradle +++ b/sdks/java/io/hadoop-common/build.gradle @@ -39,6 +39,9 @@ dependencies { compileOnly library.java.hadoop_common compileOnly library.java.hadoop_mapreduce_client_core testImplementation library.java.junit + testImplementation library.java.hadoop_client + testImplementation library.java.hadoop_common + testImplementation library.java.hadoop_mapreduce_client_core hadoopVersions.each {kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value" "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value" diff --git a/sdks/java/io/hadoop-format/build.gradle b/sdks/java/io/hadoop-format/build.gradle index e616b91aa5db..191e91526123 100644 --- a/sdks/java/io/hadoop-format/build.gradle +++ b/sdks/java/io/hadoop-format/build.gradle @@ -93,6 +93,9 @@ dependencies { testImplementation library.java.cassandra_driver_core testImplementation library.java.cassandra_driver_mapping testImplementation "org.apache.cassandra:cassandra-all:3.11.8" + testImplementation library.java.hadoop_common + testImplementation library.java.hadoop_hdfs + testImplementation library.java.hadoop_mapreduce_client_core testImplementation library.java.postgres testImplementation library.java.junit testImplementation library.java.hamcrest_core diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index 7e23ce8b0d6b..53711c6dfeff 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -92,6 +92,7 @@ dependencies { testImplementation "org.apache.hive:hive-exec:$hive_version" testImplementation "org.apache.hive:hive-common:$hive_version" testImplementation "org.apache.hive:hive-cli:$hive_version" + testImplementation "org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version" testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") hadoopVersions.each {kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value" diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index 35860b90f8a5..07ee6c186d36 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -54,6 +54,7 @@ dependencies { implementation library.java.avro // Get back to "compileOnly" since 2.14 compileOnly library.java.kafka_clients + testImplementation library.java.kafka_clients implementation library.java.slf4j_api implementation library.java.joda_time implementation library.java.jackson_annotations diff --git a/sdks/java/io/parquet/build.gradle b/sdks/java/io/parquet/build.gradle index a755dec51203..7f10f92e15d9 100644 --- a/sdks/java/io/parquet/build.gradle +++ b/sdks/java/io/parquet/build.gradle @@ -48,6 +48,7 @@ dependencies { implementation "org.apache.parquet:parquet-hadoop:$parquet_version" implementation library.java.avro compileOnly library.java.hadoop_client + testImplementation library.java.hadoop_client testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") testImplementation library.java.junit testRuntimeOnly library.java.slf4j_jdk14 From 49044d2ab982b91607020cd11ac349818ae76e6e Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 15:29:10 -0500 Subject: [PATCH 09/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- .../sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java index 706cf69762b9..b530bb7abe5c 100644 --- a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java +++ b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java @@ -22,6 +22,7 @@ import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.TEST_TABLE; import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.getExpectedRecordsAsKV; import static org.apache.beam.sdk.io.hcatalog.test.HCatalogIOTestUtils.insertTestData; +import static org.junit.Assume.assumeFalse; import java.io.IOException; import java.io.Serializable; @@ -43,6 +44,7 @@ import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.SystemUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -68,6 +70,9 @@ public class BeamSqlHiveSchemaTest implements Serializable { @BeforeClass public static void setupEmbeddedMetastoreService() throws IOException { + // TODO(BEAM-2530): Remove this when hive version 4 is released and includes + // https://github.com/apache/hive/commit/a234475faa2cab2606f2a74eb9ca071f006998e2 + assumeFalse(SystemUtils.isJavaVersionAtLeast(1.9f)); service = new EmbeddedMetastoreService(TMP_FOLDER.getRoot().getAbsolutePath()); } From 7afea6c1ac6477d47273bed86ef64b184aa9b0eb Mon Sep 17 00:00:00 2001 From: Daniel Collins Date: Tue, 21 Dec 2021 15:31:52 -0500 Subject: [PATCH 10/10] [BEAM-13430] Remove propdeps and replace with compileOnly Still generate provided in the output POM --- .../sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java index b530bb7abe5c..391016f82df6 100644 --- a/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java +++ b/sdks/java/extensions/sql/hcatalog/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/hcatalog/BeamSqlHiveSchemaTest.java @@ -70,7 +70,7 @@ public class BeamSqlHiveSchemaTest implements Serializable { @BeforeClass public static void setupEmbeddedMetastoreService() throws IOException { - // TODO(BEAM-2530): Remove this when hive version 4 is released and includes + // TODO(BEAM-13506): Remove this when hive version 4 is released and includes // https://github.com/apache/hive/commit/a234475faa2cab2606f2a74eb9ca071f006998e2 assumeFalse(SystemUtils.isJavaVersionAtLeast(1.9f)); service = new EmbeddedMetastoreService(TMP_FOLDER.getRoot().getAbsolutePath());