From 0f73897cfc369046d8b1b4badfc83516bd5bdad0 Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Fri, 7 Jan 2022 16:56:54 -0800 Subject: [PATCH 1/4] initial files for vendored gRPC 1.43.2 --- .../beam/gradle/GrpcVendoring_1_43_2.groovy | 196 ++++++++++++++++++ settings.gradle.kts | 1 + vendor/README.md | 4 +- vendor/grpc-1_43_2/build.gradle | 37 ++++ 4 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy create mode 100644 vendor/grpc-1_43_2/build.gradle diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy new file mode 100644 index 000000000000..34e26ad1a317 --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.beam.gradle + +/** + * Utilities for working with our vendored version of gRPC. + */ +class GrpcVendoring_1_43_2 { + + static def guava_version = "30.1.1-jre" + static def protobuf_version = "3.19.2" + static def grpc_version = "1.43.2" + static def gson_version = "2.8.9" + // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty + static def netty_version = "4.1.63.Final" + static def netty_ssl_version = "2.0.38.Final" + // google-auth-library version from https://search.maven.org/artifact/io.grpc/grpc-auth/1.43.2/jar + static def google_auth_version = "0.22.2" + // proto-google-common-protos version from https://search.maven.org/artifact/io.grpc/grpc-protobuf/1.43.2/jar + static def proto_google_common_protos_version = "2.0.1" + static def opencensus_version = "0.28.0" + static def conscrypt_version = "2.5.1" + + /** Returns the list of implementation time dependencies. */ + static List dependencies() { + return [ + "com.google.guava:guava:$guava_version", + "com.google.protobuf:protobuf-java:$protobuf_version", + "com.google.protobuf:protobuf-java-util:$protobuf_version", + "com.google.code.gson:gson:$gson_version", + "io.grpc:grpc-auth:$grpc_version", + "io.grpc:grpc-core:$grpc_version", + "io.grpc:grpc-context:$grpc_version", + "io.grpc:grpc-netty:$grpc_version", + "io.grpc:grpc-protobuf:$grpc_version", + "io.grpc:grpc-stub:$grpc_version", + "io.grpc:grpc-testing:$grpc_version", + // Use a classifier to ensure we get the jar containing native libraries. In the future + // hopefully netty releases a single jar containing native libraries for all architectures. + "io.netty:netty-transport-native-epoll:$netty_version:linux-x86_64", + // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty + "io.netty:netty-tcnative-boringssl-static:$netty_ssl_version", + "com.google.auth:google-auth-library-credentials:$google_auth_version", + "com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version", + "io.opencensus:opencensus-api:$opencensus_version", + "io.opencensus:opencensus-contrib-grpc-metrics:$opencensus_version", + ] + } + + /** + * Returns the list of runtime time dependencies that should be exported as runtime + * dependencies within the vendored jar. + */ + static List runtimeDependencies() { + return [ + 'com.google.errorprone:error_prone_annotations:2.9.0', + // TODO(BEAM-9288): Enable relocation for conscrypt + "org.conscrypt:conscrypt-openjdk-uber:$conscrypt_version" + ] + } + + /** + * Returns the list of test dependencies. + */ + static List testDependencies() { + return [ + 'junit:junit:4.12', + ] + } + + static Map relocations() { + // The relocation paths below specifically use gRPC and the full version string as + // the code relocation prefix. See https://lists.apache.org/thread.html/4c12db35b40a6d56e170cd6fc8bb0ac4c43a99aa3cb7dbae54176815@%3Cdev.beam.apache.org%3E + // for further details. + + // To produce the list of necessary relocations, one needs to start with a set of target + // packages that one wants to vendor, find all necessary transitive dependencies of that + // set and provide relocations for each such that all necessary packages and their + // dependencies are relocated. Any optional dependency that doesn't need relocation + // must be excluded via an 'exclude' rule. There is additional complexity of libraries that use + // JNI or reflection and have to be handled on case by case basis by learning whether + // they support relocation and how would one go about doing it by reading any documentation + // those libraries may provide. The 'validateShadedJarDoesntLeakNonOrgApacheBeamClasses' + // ensures that there are no classes outside of the 'org.apache.beam' namespace. + + String version = "v1p43p2"; + String prefix = "org.apache.beam.vendor.grpc.${version}"; + List packagesToRelocate = [ + // guava uses the com.google.common and com.google.thirdparty package namespaces + "com.google.common", + "com.google.thirdparty", + "com.google.protobuf", + "com.google.gson", + "com.google.auth", + "com.google.api", + "com.google.cloud", + "com.google.logging", + "com.google.longrunning", + "com.google.rpc", + "com.google.type", + "com.google.geo.type", + "io.grpc", + "io.netty", + "io.opencensus", + "io.perfmark", + ] + + return packagesToRelocate.collectEntries { + [ (it): "${prefix}.${it}" ] + } + [ + // Adapted from https://github.com/grpc/grpc-java/blob/e283f70ad91f99c7fee8b31b605ef12a4f9b1690/netty/shaded/build.gradle#L41 + // We "io.netty": "${prefix}.io.netty",have to be careful with these replacements as they must not match any + // string in NativeLibraryLoader, else they cause corruption. Note that + // this includes concatenation of string literals and constants. + 'META-INF/native/libnetty': "META-INF/native/liborg_apache_beam_vendor_grpc_${version}_netty", + 'META-INF/native/netty': "META-INF/native/org_apache_beam_vendor_grpc_${version}_netty", + 'META-INF/native/lib-netty': "META-INF/native/lib-org-apache-beam-vendor-grpc-${version}-netty", + ] + } + + /** Returns the list of shading exclusions. */ + static List exclusions() { + return [ + // Don't include in the vendored jar: + // android annotations, errorprone, checkerframework, JDK8 annotations, objenesis, junit, + // commons-logging, log4j, slf4j and mockito + "android/annotation/**/", + "com/google/errorprone/**", + "com/google/instrumentation/**", + "com/google/j2objc/annotations/**", + "io/netty/handler/codec/marshalling/**", + "io/netty/handler/codec/spdy/**", + "io/netty/handler/codec/compression/JZlib*", + "io/netty/handler/codec/compression/Lz4*", + "io/netty/handler/codec/compression/Lzf*", + "io/netty/handler/codec/compression/Lzma*", + "io/netty/handler/codec/protobuf/Protobuf*Nano.class", + "io/netty/util/internal/logging/CommonsLogger*", + "io/netty/util/internal/logging/LocationAwareSlf4JLogger*", + "io/netty/util/internal/logging/Log4JLogger*", + "io/netty/util/internal/logging/Log4J2Logger*", + "javax/annotation/**", + "junit/**", + "module-info.class", + "org/checkerframework/**", + "org/codehaus/mojo/animal_sniffer/**", + "org/conscrypt/**", + "META-INF/native/libconscrypt**", + "META-INF/native/conscrypt**", + "org/hamcrest/**", + "org/junit/**", + "org/mockito/**", + "org/objenesis/**", + ] + } + + /** + * Returns a closure contaning the dependencies map used for shading gRPC within the main + * Apache Beam project. + */ + static Object dependenciesClosure() { + return { + dependencies().each { implementation it } + runtimeDependencies().each { shadow it } + } + } + + /** + * Returns a closure with the code relocation configuration for shading gRPC within the main + * Apache Beam project. + */ + static Object shadowClosure() { + return { + relocations().each { srcNamespace, destNamespace -> + relocate srcNamespace, destNamespace + } + exclusions().each { exclude it } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index f36c645fd0e4..bc8576a01d86 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -218,6 +218,7 @@ include(":sdks:python:test-suites:tox:py36") include(":sdks:python:test-suites:tox:py37") include(":sdks:python:test-suites:tox:py38") include(":vendor:grpc-1_36_0") +include(":vendor:grpc-1_43_2") include(":vendor:bytebuddy-1_11_0") include(":vendor:calcite-1_28_0") include(":vendor:guava-26_0-jre") diff --git a/vendor/README.md b/vendor/README.md index 3bef746d3c2e..a552494bb300 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -61,8 +61,8 @@ As long as Beam's use of gRPC does not touch these optional Netty features or th available at runtime, it's fine to have the references to the missing classes. Here are the known linkage errors: -- References to `org.junit.runners`: `io.grpc.testing.GrpcCleanupRule` uses JUnit classes, which are - present when we run Beam's tests. +- References to `org.junit.*`: `io.grpc.testing.GrpcCleanupRule` and `io.grpc.testing.GrpcServerRule` + uses JUnit classes, which are present when we run Beam's tests. - References from `io.netty.handler.ssl`: Netty users can choose SSL implementation based on the platform ([Netty documentation](https://netty.io/wiki/forked-tomcat-native.html#wiki-h2-4)). Beam's vendored gRPC uses `netty-tcnative-boringssl-static`, which contains the static libraries diff --git a/vendor/grpc-1_43_2/build.gradle b/vendor/grpc-1_43_2/build.gradle new file mode 100644 index 000000000000..b1d8b84b5dc6 --- /dev/null +++ b/vendor/grpc-1_43_2/build.gradle @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.beam.gradle.GrpcVendoring_1_43_2 + +plugins { id 'org.apache.beam.vendor-java' } + +description = "Apache Beam :: Vendored Dependencies :: gRPC :: 1.43.2" + +group = "org.apache.beam" +version = "0.1" + +vendorJava( + dependencies: GrpcVendoring_1_43_2.dependencies(), + runtimeDependencies: GrpcVendoring_1_43_2.runtimeDependencies(), + testDependencies: GrpcVendoring_1_43_2.testDependencies(), + relocations: GrpcVendoring_1_43_2.relocations(), + exclusions: GrpcVendoring_1_43_2.exclusions(), + artifactId: "beam-vendor-grpc-1_43_2", + groupId: group, + version: version, +) From e9a8725ebff14641ec3d9ad791c614249bf5cc65 Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Tue, 11 Jan 2022 09:55:57 -0800 Subject: [PATCH 2/4] update comments and ordering of variables --- .../beam/gradle/GrpcVendoring_1_43_2.groovy | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy index 34e26ad1a317..2ccc7575ab97 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy @@ -20,22 +20,29 @@ package org.apache.beam.gradle /** * Utilities for working with our vendored version of gRPC. + * + * To update: + * 1. Determine the set of io.grpc libraries we want to include. + * 2. Use mvn dependency:tree and https://search.maven.org/search?q=g:io.grpc + * to determine dependency tree. You may need to search for optional dependencies + * and determine if they are needed (e.g. conscrypt). */ class GrpcVendoring_1_43_2 { + static def grpc_version = "1.43.2" + // See https://github.com/grpc/grpc-java/blob/v1.43.2/build.gradle + // or https://search.maven.org/search?q=g:io.grpc%201.43.2 static def guava_version = "30.1.1-jre" static def protobuf_version = "3.19.2" - static def grpc_version = "1.43.2" static def gson_version = "2.8.9" - // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty - static def netty_version = "4.1.63.Final" - static def netty_ssl_version = "2.0.38.Final" - // google-auth-library version from https://search.maven.org/artifact/io.grpc/grpc-auth/1.43.2/jar static def google_auth_version = "0.22.2" - // proto-google-common-protos version from https://search.maven.org/artifact/io.grpc/grpc-protobuf/1.43.2/jar - static def proto_google_common_protos_version = "2.0.1" static def opencensus_version = "0.28.0" static def conscrypt_version = "2.5.1" + static def proto_google_common_protos_version = "2.0.1" + + // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty + static def netty_version = "4.1.63.Final" + static def netty_ssl_version = "2.0.38.Final" /** Returns the list of implementation time dependencies. */ static List dependencies() { From 32bbd4389feceaf709d33deb4082cffcc1228376 Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Tue, 11 Jan 2022 11:11:41 -0800 Subject: [PATCH 3/4] comments and rename variable --- .../apache/beam/gradle/GrpcVendoring_1_43_2.groovy | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy index 2ccc7575ab97..7fbe2252e45e 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy @@ -22,10 +22,15 @@ package org.apache.beam.gradle * Utilities for working with our vendored version of gRPC. * * To update: - * 1. Determine the set of io.grpc libraries we want to include. + * 1. Determine the set of io.grpc libraries we want to include, most likely a superset of + * of the previous vendored gRPC version. * 2. Use mvn dependency:tree and https://search.maven.org/search?q=g:io.grpc * to determine dependency tree. You may need to search for optional dependencies * and determine if they are needed (e.g. conscrypt). + * 3. Validate built artifacts by running linkage tool + * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) + * and unit and integration tests in a PR (e.g. for https://github.com/apache/beam/pull/16460, + * https://github.com/apache/beam/pull/16459) */ class GrpcVendoring_1_43_2 { static def grpc_version = "1.43.2" @@ -42,7 +47,7 @@ class GrpcVendoring_1_43_2 { // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty static def netty_version = "4.1.63.Final" - static def netty_ssl_version = "2.0.38.Final" + static def netty_tcnative_version = "2.0.38.Final" /** Returns the list of implementation time dependencies. */ static List dependencies() { @@ -61,8 +66,7 @@ class GrpcVendoring_1_43_2 { // Use a classifier to ensure we get the jar containing native libraries. In the future // hopefully netty releases a single jar containing native libraries for all architectures. "io.netty:netty-transport-native-epoll:$netty_version:linux-x86_64", - // tcnative version from https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty - "io.netty:netty-tcnative-boringssl-static:$netty_ssl_version", + "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version", "com.google.auth:google-auth-library-credentials:$google_auth_version", "com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version", "io.opencensus:opencensus-api:$opencensus_version", From 9bc949dc50c5c5a6c4cfbc84294e6010bdf14042 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Tue, 11 Jan 2022 11:24:59 -0800 Subject: [PATCH 4/4] Update buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy --- .../groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy index 7fbe2252e45e..c05e9ecb7db4 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_43_2.groovy @@ -29,7 +29,7 @@ package org.apache.beam.gradle * and determine if they are needed (e.g. conscrypt). * 3. Validate built artifacts by running linkage tool * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) - * and unit and integration tests in a PR (e.g. for https://github.com/apache/beam/pull/16460, + * and unit and integration tests in a PR (e.g. https://github.com/apache/beam/pull/16460, * https://github.com/apache/beam/pull/16459) */ class GrpcVendoring_1_43_2 {