From 310caf34ef7224f9778ba5f142ea0851f42d7186 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Fri, 10 Apr 2020 17:03:21 -0400 Subject: [PATCH] Applying difference --- .../beam/gradle/BeamModulePlugin.groovy | 107 +++++++++++------- examples/java/build.gradle | 1 + .../google-cloud-dataflow-java/build.gradle | 1 + .../worker/build.gradle | 1 + .../google-cloud-platform-core/build.gradle | 1 + .../extensions/sql/datacatalog/build.gradle | 1 + sdks/java/extensions/sql/zetasql/build.gradle | 1 + .../io/google-cloud-platform/build.gradle | 1 + 8 files changed, 74 insertions(+), 40 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 477fb40d618c..14e35a833a36 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -22,6 +22,8 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import groovy.json.JsonOutput import groovy.json.JsonSlurper import java.util.concurrent.atomic.AtomicInteger +import org.gradle.api.attributes.Attribute +import org.gradle.api.attributes.AttributeContainer import org.gradle.api.GradleException import org.gradle.api.Plugin import org.gradle.api.Project @@ -373,15 +375,9 @@ class BeamModulePlugin implements Plugin { def aws_java_sdk2_version = "2.10.61" def cassandra_driver_version = "3.8.0" def classgraph_version = "4.8.65" - def gax_version = "1.54.0" - def generated_grpc_ga_version = "1.85.1" - def google_auth_version = "0.19.0" def google_clients_version = "1.30.3" - def google_cloud_bigdataoss_version = "2.0.0" - def google_cloud_core_version = "1.92.2" - def google_cloud_spanner_version = "1.49.1" - def google_cloud_datacatalog_version = "0.32.1" - def google_http_clients_version = "1.34.0" + def google_cloud_bigdataoss_version = "2.1.2" + // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom def grpc_version = "1.27.2" def guava_version = "25.1-jre" def hadoop_version = "2.8.5" @@ -393,7 +389,6 @@ class BeamModulePlugin implements Plugin { def netty_version = "4.1.30.Final" def postgres_version = "42.2.2" def powermock_version = "2.0.2" - def proto_google_common_protos_version = "1.17.0" def protobuf_version = "3.11.1" def quickcheck_version = "0.8" def spark_version = "2.4.5" @@ -444,45 +439,48 @@ class BeamModulePlugin implements Plugin { commons_lang3 : "org.apache.commons:commons-lang3:3.9", commons_math3 : "org.apache.commons:commons-math3:3.6.1", error_prone_annotations : "com.google.errorprone:error_prone_annotations:2.0.15", - gax : "com.google.api:gax:$gax_version", - gax_grpc : "com.google.api:gax-grpc:$gax_version", + gax : "com.google.api:gax", // google_cloud_platform_libraries_bom sets version + gax_grpc : "com.google.api:gax-grpc", // google_cloud_platform_libraries_bom sets version google_api_client : "com.google.api-client:google-api-client:$google_clients_version", google_api_client_jackson2 : "com.google.api-client:google-api-client-jackson2:$google_clients_version", google_api_client_java6 : "com.google.api-client:google-api-client-java6:$google_clients_version", - google_api_common : "com.google.api:api-common:1.8.1", + google_api_common : "com.google.api:api-common", // google_cloud_platform_libraries_bom sets version google_api_services_bigquery : "com.google.apis:google-api-services-bigquery:v2-rev20191211-$google_clients_version", google_api_services_clouddebugger : "com.google.apis:google-api-services-clouddebugger:v2-rev20191003-$google_clients_version", google_api_services_cloudresourcemanager : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20191206-$google_clients_version", google_api_services_dataflow : "com.google.apis:google-api-services-dataflow:v1b3-rev20190927-$google_clients_version", google_api_services_pubsub : "com.google.apis:google-api-services-pubsub:v1-rev20191111-$google_clients_version", google_api_services_storage : "com.google.apis:google-api-services-storage:v1-rev20191011-$google_clients_version", - google_auth_library_credentials : "com.google.auth:google-auth-library-credentials:$google_auth_version", - google_auth_library_oauth2_http : "com.google.auth:google-auth-library-oauth2-http:$google_auth_version", - google_cloud_bigquery : "com.google.cloud:google-cloud-bigquery:1.108.0", - google_cloud_bigquery_storage : "com.google.cloud:google-cloud-bigquerystorage:0.125.0-beta", + google_auth_library_credentials : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version + google_auth_library_oauth2_http : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version + google_cloud_bigquery : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version + google_cloud_bigquery_storage : "com.google.cloud:google-cloud-bigquerystorage", // google_cloud_platform_libraries_bom sets version google_cloud_bigtable_client_core : "com.google.cloud.bigtable:bigtable-client-core:1.13.0", - google_cloud_core : "com.google.cloud:google-cloud-core:$google_cloud_core_version", - google_cloud_core_grpc : "com.google.cloud:google-cloud-core-grpc:$google_cloud_core_version", - google_cloud_datacatalog_v1beta1 : "com.google.cloud:google-cloud-datacatalog:$google_cloud_datacatalog_version", + google_cloud_core : "com.google.cloud:google-cloud-core", // google_cloud_platform_libraries_bom sets version + google_cloud_core_grpc : "com.google.cloud:google-cloud-core-grpc", // google_cloud_platform_libraries_bom sets version + google_cloud_datacatalog_v1beta1 : "com.google.cloud:google-cloud-datacatalog", // google_cloud_platform_libraries_bom sets version google_cloud_dataflow_java_proto_library_all: "com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304", google_cloud_datastore_v1_proto_client : "com.google.cloud.datastore:datastore-v1-proto-client:1.6.3", - google_cloud_spanner : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version", - google_http_client : "com.google.http-client:google-http-client:$google_http_clients_version", + // The GCP Libraries BOM dashboard shows the versions set by the BOM: + // https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/4.2.0/artifact_details.html + google_cloud_platform_libraries_bom : "com.google.cloud:libraries-bom:4.2.0", + google_cloud_spanner : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version + // google-http-client's version is explicitly declared for sdks/java/maven-archetypes/examples + // This version should be in line with the one in com.google.cloud:libraries-bom. + google_http_client : "com.google.http-client:google-http-client:1.34.2", // google_cloud_platform_libraries_bom sets version google_http_client_jackson : "com.google.http-client:google-http-client-jackson:1.29.2", - google_http_client_jackson2 : "com.google.http-client:google-http-client-jackson2:$google_http_clients_version", - google_http_client_protobuf : "com.google.http-client:google-http-client-protobuf:$google_http_clients_version", + google_http_client_jackson2 : "com.google.http-client:google-http-client-jackson2", // google_cloud_platform_libraries_bom sets version + google_http_client_protobuf : "com.google.http-client:google-http-client-protobuf", // google_cloud_platform_libraries_bom sets version google_oauth_client : "com.google.oauth-client:google-oauth-client:$google_clients_version", google_oauth_client_java6 : "com.google.oauth-client:google-oauth-client-java6:$google_clients_version", grpc_all : "io.grpc:grpc-all:$grpc_version", - grpc_auth : "io.grpc:grpc-auth:$grpc_version", - grpc_core : "io.grpc:grpc-core:$grpc_version", - grpc_context : "io.grpc:grpc-context:$grpc_version", - grpc_google_cloud_pubsub_v1 : "com.google.api.grpc:grpc-google-cloud-pubsub-v1:$generated_grpc_ga_version", - grpc_grpclb : "io.grpc:grpc-grpclb:$grpc_version", - grpc_protobuf : "io.grpc:grpc-protobuf:$grpc_version", - grpc_protobuf_lite : "io.grpc:grpc-protobuf-lite:$grpc_version", - grpc_netty : "io.grpc:grpc-netty:$grpc_version", - grpc_stub : "io.grpc:grpc-stub:$grpc_version", + grpc_auth : "io.grpc:grpc-auth", // google_cloud_platform_libraries_bom sets version + grpc_core : "io.grpc:grpc-core", // google_cloud_platform_libraries_bom sets version + grpc_context : "io.grpc:grpc-context", // google_cloud_platform_libraries_bom sets version + grpc_google_cloud_pubsub_v1 : "com.google.api.grpc:grpc-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version + grpc_grpclb : "io.grpc:grpc-grpclb", // google_cloud_platform_libraries_bom sets version + grpc_netty : "io.grpc:grpc-netty", // google_cloud_platform_libraries_bom sets version + grpc_stub : "io.grpc:grpc-stub", // google_cloud_platform_libraries_bom sets version guava : "com.google.guava:guava:$guava_version", guava_testlib : "com.google.guava:guava-testlib:$guava_version", hadoop_client : "org.apache.hadoop:hadoop-client:$hadoop_version", @@ -521,12 +519,12 @@ class BeamModulePlugin implements Plugin { powermock_mockito : "org.powermock:powermock-api-mockito2:$powermock_version", protobuf_java : "com.google.protobuf:protobuf-java:$protobuf_version", protobuf_java_util : "com.google.protobuf:protobuf-java-util:$protobuf_version", - proto_google_cloud_bigquery_storage_v1beta1 : "com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.85.1", - proto_google_cloud_bigtable_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-v2:1.9.1", - proto_google_cloud_datastore_v1 : "com.google.api.grpc:proto-google-cloud-datastore-v1:0.85.0", - proto_google_cloud_pubsub_v1 : "com.google.api.grpc:proto-google-cloud-pubsub-v1:$generated_grpc_ga_version", - proto_google_cloud_spanner_admin_database_v1: "com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:$google_cloud_spanner_version", - proto_google_common_protos : "com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version", + proto_google_cloud_bigquery_storage_v1beta1 : "com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1", // google_cloud_platform_libraries_bom sets version + proto_google_cloud_bigtable_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-v2", // google_cloud_platform_libraries_bom sets version + proto_google_cloud_datastore_v1 : "com.google.api.grpc:proto-google-cloud-datastore-v1", // google_cloud_platform_libraries_bom sets version + proto_google_cloud_pubsub_v1 : "com.google.api.grpc:proto-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version + proto_google_cloud_spanner_admin_database_v1: "com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1", // google_cloud_platform_libraries_bom sets version + proto_google_common_protos : "com.google.api.grpc:proto-google-common-protos", // google_cloud_platform_libraries_bom sets version slf4j_api : "org.slf4j:slf4j-api:1.7.25", slf4j_simple : "org.slf4j:slf4j-simple:1.7.25", slf4j_jdk14 : "org.slf4j:slf4j-jdk14:1.7.25", @@ -1160,8 +1158,20 @@ class BeamModulePlugin implements Plugin { } def dependenciesNode = root.appendNode('dependencies') + + // BOMs, declared with 'platform', appear in section + def boms = [] + + // This value is from PlatformSupport.COMPONENT_CATEGORY (Gradle's internal API) + def componentCategory = Attribute.of('org.gradle.component.category', java.lang.String.class); def generateDependenciesFromConfiguration = { param -> project.configurations."${param.configuration}".allDependencies.each { + AttributeContainer attributes = it.getAttributes() + if (attributes.getAttribute(componentCategory) == 'platform') { + boms.add(it); + return + } + def dependencyNode = dependenciesNode.appendNode('dependency') def appendClassifier = { dep -> dep.artifacts.each { art -> @@ -1180,7 +1190,9 @@ class BeamModulePlugin implements Plugin { } else { dependencyNode.appendNode('groupId', it.group) dependencyNode.appendNode('artifactId', it.name) - dependencyNode.appendNode('version', it.version) + if (it.version != null) { // bom-managed artifacts do not have their versions + dependencyNode.appendNode('version', it.version) + } dependencyNode.appendNode('scope', param.scope) appendClassifier(it) } @@ -1210,6 +1222,18 @@ class BeamModulePlugin implements Plugin { configuration: (configuration.shadowClosure ? 'shadow' : 'compile'), scope: 'compile') generateDependenciesFromConfiguration(configuration: 'provided', scope: 'provided') + if (!boms.isEmpty()) { + def dependencyManagementNode = root.appendNode('dependencyManagement') + def dependencyManagementDependencies = dependencyManagementNode.appendNode('dependencies') + boms.each { + def dependencyNode = dependencyManagementDependencies.appendNode('dependency') + dependencyNode.appendNode('groupId', it.group) + dependencyNode.appendNode('artifactId', it.name) + dependencyNode.appendNode('version', it.version) + dependencyNode.appendNode('type', 'pom') + dependencyNode.appendNode('scope', 'import') + } + } // NB: This must come after asNode() logic, as it seems asNode() // removes XML comments. // TODO: Load this from file? @@ -1274,7 +1298,10 @@ class BeamModulePlugin implements Plugin { // has different dependencies than our project. if (config.getName() != "errorprone" && !inDependencyUpdates) { config.resolutionStrategy { - force project.library.java.values() + // Filtering versionless coordinates that depend on BOM. Beam project needs to set the + // versions for only handful libraries when building the project (BEAM-9542). + def librariesWithVersion = project.library.java.values().findAll { it.split(':').size() > 2 } + force librariesWithVersion } } } diff --git a/examples/java/build.gradle b/examples/java/build.gradle index 5b08348aa5c8..04f25528c771 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -45,6 +45,7 @@ configurations.sparkRunnerPreCommit { } dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile library.java.vendored_guava_26_0_jre compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:google-cloud-platform-core") diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 48a34d8f75d5..1bac782798d7 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -60,6 +60,7 @@ configurations { } dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile library.java.vendored_guava_26_0_jre compile project(path: ":model:pipeline", configuration: "shadow") compile project(path: ":sdks:java:core", configuration: "shadow") diff --git a/runners/google-cloud-dataflow-java/worker/build.gradle b/runners/google-cloud-dataflow-java/worker/build.gradle index a0e335632bc5..546e6f8919dd 100644 --- a/runners/google-cloud-dataflow-java/worker/build.gradle +++ b/runners/google-cloud-dataflow-java/worker/build.gradle @@ -65,6 +65,7 @@ dependencies { // // All main sourceset dependencies here should be listed as compile scope so that the dependencies // are all packaged into a single uber jar allowing the jar to serve as an application. + compile platform(library.java.google_cloud_platform_libraries_bom) compile project(":runners:google-cloud-dataflow-java") compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:google-cloud-platform-core") diff --git a/sdks/java/extensions/google-cloud-platform-core/build.gradle b/sdks/java/extensions/google-cloud-platform-core/build.gradle index 9ffa22902a47..cd09867e0bd8 100644 --- a/sdks/java/extensions/google-cloud-platform-core/build.gradle +++ b/sdks/java/extensions/google-cloud-platform-core/build.gradle @@ -34,6 +34,7 @@ test { } dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile library.java.vendored_guava_26_0_jre compile project(path: ":sdks:java:core", configuration: "shadow") compile library.java.google_http_client_jackson2 diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 5068c8bde80a..f49783032da2 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -23,6 +23,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.datacatalog') dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile(library.java.google_cloud_datacatalog_v1beta1) { exclude group: 'io.grpc', module: 'grpc-core' // Use Beam's version } diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index fd5364187700..240f251261a0 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -28,6 +28,7 @@ ext.summary = "ZetaSQL to Calcite translator" def zetasql_version = "2020.03.2" dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile project(":sdks:java:core") compile project(":sdks:java:extensions:sql") compile library.java.vendored_calcite_1_20_0 diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle index 66db0bea448c..bac1cdd1fe60 100644 --- a/sdks/java/io/google-cloud-platform/build.gradle +++ b/sdks/java/io/google-cloud-platform/build.gradle @@ -28,6 +28,7 @@ description = "Apache Beam :: SDKs :: Java :: IO :: Google Cloud Platform" ext.summary = "IO library to read and write Google Cloud Platform systems from Beam." dependencies { + compile platform(library.java.google_cloud_platform_libraries_bom) compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:expansion-service") compile project(":sdks:java:extensions:google-cloud-platform-core")