From 2caf4d6df08334c4bd86c1f5bbfc4ee6fcae644e Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Mon, 10 Jan 2022 15:31:16 -0800 Subject: [PATCH 1/4] update protobuf on calcite --- vendor/calcite-1_28_0/build.gradle | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vendor/calcite-1_28_0/build.gradle b/vendor/calcite-1_28_0/build.gradle index adf154bc6fd5..777d48fd7964 100644 --- a/vendor/calcite-1_28_0/build.gradle +++ b/vendor/calcite-1_28_0/build.gradle @@ -21,16 +21,20 @@ plugins { id 'org.apache.beam.vendor-java' } description = "Apache Beam :: Vendored Dependencies :: Calcite 1.28.0" group = "org.apache.beam" -version = "0.1" +version = "0.2" def calcite_version = "1.28.0" def avatica_version = "1.19.0" + +// TODO(BEAM-13616): Calcite does not automatically use an up-to-date protobuf dependency. +def protobuf_version = "3.19.2" def prefix = "org.apache.beam.vendor.calcite.v1_28_0" List packagesToRelocate = [ "com.esri", "com.fasterxml", "com.google.common", + "com.google.gson", "com.google.protobuf", "com.google.thirdparty", "com.google.uzaygezen", @@ -52,6 +56,8 @@ vendorJava( "org.apache.calcite:calcite-core:$calcite_version", "org.apache.calcite:calcite-linq4j:$calcite_version", "org.apache.calcite.avatica:avatica-core:$avatica_version", + "com.google.protobuf:protobuf-java:$protobuf_version", + "com.google.protobuf:protobuf-java-util:$protobuf_version", ], runtimeDependencies: [ library.java.slf4j_api, From 832dc3dbfae40c6caefbc562abecb80324623262 Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Tue, 11 Jan 2022 15:59:53 -0800 Subject: [PATCH 2/4] comments --- vendor/calcite-1_28_0/build.gradle | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/vendor/calcite-1_28_0/build.gradle b/vendor/calcite-1_28_0/build.gradle index 777d48fd7964..f6b3ee5bf5e7 100644 --- a/vendor/calcite-1_28_0/build.gradle +++ b/vendor/calcite-1_28_0/build.gradle @@ -16,6 +16,20 @@ * limitations under the License. */ +/** + * Vendored version of calcite. + * + * To upgrade: + * 1. Use mvn dependency:tree and/or https://search.maven.org/search?q=g:org.apache.calcite%20AND%20a:calcite-core + * to determine dependency tree. You may need to search for optional transitive dependencies + * and determine if they need to be added or upgraded (e.g. protobuf) + * 3. Validate built artifacts by running linkage tool + * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) + * and unit and integration tests in a PR. Example PRs: + * * A version upgrade for gRPC https://github.com/apache/beam/pull/16460 + * * A minor version to vendored package: https://github.com/apache/beam/pull/16476 + */ + plugins { id 'org.apache.beam.vendor-java' } description = "Apache Beam :: Vendored Dependencies :: Calcite 1.28.0" @@ -25,8 +39,6 @@ version = "0.2" def calcite_version = "1.28.0" def avatica_version = "1.19.0" - -// TODO(BEAM-13616): Calcite does not automatically use an up-to-date protobuf dependency. def protobuf_version = "3.19.2" def prefix = "org.apache.beam.vendor.calcite.v1_28_0" @@ -56,6 +68,10 @@ vendorJava( "org.apache.calcite:calcite-core:$calcite_version", "org.apache.calcite:calcite-linq4j:$calcite_version", "org.apache.calcite.avatica:avatica-core:$avatica_version", + + // BEAM-13616: Override the version of protobuf to patch a security vulnerability. + // This override can be removed once we upgrade to a newer version of calcite that + // depends on protobuf >= 3.19.2. "com.google.protobuf:protobuf-java:$protobuf_version", "com.google.protobuf:protobuf-java-util:$protobuf_version", ], From bad9151105570fff5148c64f550d337bc66b6bfe Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Tue, 11 Jan 2022 16:20:03 -0800 Subject: [PATCH 3/4] add guide in README.md --- vendor/README.md | 38 +++++++++++++++++++++++++++++- vendor/calcite-1_28_0/build.gradle | 4 +--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/vendor/README.md b/vendor/README.md index a552494bb300..d8df865a67b7 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -28,6 +28,7 @@ The upgrading of the vendored dependencies should be performed in two steps: # How to validate the vendored dependencies +## Linkage Tool The [linkage tool](https://lists.apache.org/thread.html/eb5d95b9a33d7e32dc9bcd0f7d48ba8711d42bd7ed03b9cf0f1103f1%40%3Cdev.beam.apache.org%3E) is useful for the vendored dependency upgrades. It reports the linkage errors across multiple Apache Beam artifact ids. @@ -48,7 +49,7 @@ $ mvn install:install-file \ $ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_36_0:0.1 :checkJavaLinkage ``` -## Known Linkage Errors in the Vendored gRPC Dependencies +### Known Linkage Errors in the Vendored gRPC Dependencies It's expected that the task outputs some linkage errors. While the `checkJavaLinkage` task does not retrieve optional dependencies to avoid bloated @@ -78,3 +79,38 @@ references to the missing classes. Here are the known linkage errors: to be included in the vendored artifact. Slf4j-api is available at Beam's runtime. - References to `reactor.blockhound`: When enabled, Netty's BlockHound integration can detect unexpected blocking calls. Beam does not use it. + +## Create testing PR against new artifacts + +Once you've verified using the linkage tool, you can test new artifacts by running unit and integration tests against a PR. + +Example PRs: +- Updating gRPC version (large) https://github.com/apache/beam/pull/16460 +- Updating protobuf for calcite (minor version update): https://github.com/apache/beam/pull/16476 + +Steps: + +1. Generate new artifact files with `publishMavenJavaPublicationToMavenLocal`, e.g. + +``` +./gradlew -p vendor/grpc-1_43_2 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly + +# Copy files (jar/poms/metadata) to your beam repository +cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_43_2/ \ + $BEAMDIR/tempLib/org/apache/beam/beam-vendor-grpc-1_43_2 +``` + +2. Add whatever folder (here I use `tempLib`) to the expected project repositories, e.g. + +``` +repositories { + maven { url "${project.rootDir}/tempLib" } + maven { + ... + } +} +``` + +3. Migrate all references from the old dependency to the new dependency, including imports if needed. + +4. Commit any added or changed files and create a PR (can be a draft, as you will not merge this PR) to test on. diff --git a/vendor/calcite-1_28_0/build.gradle b/vendor/calcite-1_28_0/build.gradle index f6b3ee5bf5e7..e2e75b45aebf 100644 --- a/vendor/calcite-1_28_0/build.gradle +++ b/vendor/calcite-1_28_0/build.gradle @@ -25,9 +25,7 @@ * and determine if they need to be added or upgraded (e.g. protobuf) * 3. Validate built artifacts by running linkage tool * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) - * and unit and integration tests in a PR. Example PRs: - * * A version upgrade for gRPC https://github.com/apache/beam/pull/16460 - * * A minor version to vendored package: https://github.com/apache/beam/pull/16476 + * and unit and integration tests in a PR. */ plugins { id 'org.apache.beam.vendor-java' } From 76d21189b034490b516453dc9c5be9f79aebb671 Mon Sep 17 00:00:00 2001 From: Emily Ye Date: Tue, 11 Jan 2022 23:11:33 -0800 Subject: [PATCH 4/4] whitespace --- vendor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/README.md b/vendor/README.md index d8df865a67b7..e5eddba98b7d 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -111,6 +111,6 @@ repositories { } ``` -3. Migrate all references from the old dependency to the new dependency, including imports if needed. +3. Migrate all references from the old dependency to the new dependency, including imports if needed. 4. Commit any added or changed files and create a PR (can be a draft, as you will not merge this PR) to test on.