diff --git a/vendor/README.md b/vendor/README.md index a552494bb300..e5eddba98b7d 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -28,6 +28,7 @@ The upgrading of the vendored dependencies should be performed in two steps: # How to validate the vendored dependencies +## Linkage Tool The [linkage tool](https://lists.apache.org/thread.html/eb5d95b9a33d7e32dc9bcd0f7d48ba8711d42bd7ed03b9cf0f1103f1%40%3Cdev.beam.apache.org%3E) is useful for the vendored dependency upgrades. It reports the linkage errors across multiple Apache Beam artifact ids. @@ -48,7 +49,7 @@ $ mvn install:install-file \ $ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_36_0:0.1 :checkJavaLinkage ``` -## Known Linkage Errors in the Vendored gRPC Dependencies +### Known Linkage Errors in the Vendored gRPC Dependencies It's expected that the task outputs some linkage errors. While the `checkJavaLinkage` task does not retrieve optional dependencies to avoid bloated @@ -78,3 +79,38 @@ references to the missing classes. Here are the known linkage errors: to be included in the vendored artifact. Slf4j-api is available at Beam's runtime. - References to `reactor.blockhound`: When enabled, Netty's BlockHound integration can detect unexpected blocking calls. Beam does not use it. + +## Create testing PR against new artifacts + +Once you've verified using the linkage tool, you can test new artifacts by running unit and integration tests against a PR. + +Example PRs: +- Updating gRPC version (large) https://github.com/apache/beam/pull/16460 +- Updating protobuf for calcite (minor version update): https://github.com/apache/beam/pull/16476 + +Steps: + +1. Generate new artifact files with `publishMavenJavaPublicationToMavenLocal`, e.g. + +``` +./gradlew -p vendor/grpc-1_43_2 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly + +# Copy files (jar/poms/metadata) to your beam repository +cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_43_2/ \ + $BEAMDIR/tempLib/org/apache/beam/beam-vendor-grpc-1_43_2 +``` + +2. Add whatever folder (here I use `tempLib`) to the expected project repositories, e.g. + +``` +repositories { + maven { url "${project.rootDir}/tempLib" } + maven { + ... + } +} +``` + +3. Migrate all references from the old dependency to the new dependency, including imports if needed. + +4. Commit any added or changed files and create a PR (can be a draft, as you will not merge this PR) to test on. diff --git a/vendor/calcite-1_28_0/build.gradle b/vendor/calcite-1_28_0/build.gradle index adf154bc6fd5..e2e75b45aebf 100644 --- a/vendor/calcite-1_28_0/build.gradle +++ b/vendor/calcite-1_28_0/build.gradle @@ -16,21 +16,35 @@ * limitations under the License. */ +/** + * Vendored version of calcite. + * + * To upgrade: + * 1. Use mvn dependency:tree and/or https://search.maven.org/search?q=g:org.apache.calcite%20AND%20a:calcite-core + * to determine dependency tree. You may need to search for optional transitive dependencies + * and determine if they need to be added or upgraded (e.g. protobuf) + * 3. Validate built artifacts by running linkage tool + * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) + * and unit and integration tests in a PR. + */ + plugins { id 'org.apache.beam.vendor-java' } description = "Apache Beam :: Vendored Dependencies :: Calcite 1.28.0" group = "org.apache.beam" -version = "0.1" +version = "0.2" def calcite_version = "1.28.0" def avatica_version = "1.19.0" +def protobuf_version = "3.19.2" def prefix = "org.apache.beam.vendor.calcite.v1_28_0" List packagesToRelocate = [ "com.esri", "com.fasterxml", "com.google.common", + "com.google.gson", "com.google.protobuf", "com.google.thirdparty", "com.google.uzaygezen", @@ -52,6 +66,12 @@ vendorJava( "org.apache.calcite:calcite-core:$calcite_version", "org.apache.calcite:calcite-linq4j:$calcite_version", "org.apache.calcite.avatica:avatica-core:$avatica_version", + + // BEAM-13616: Override the version of protobuf to patch a security vulnerability. + // This override can be removed once we upgrade to a newer version of calcite that + // depends on protobuf >= 3.19.2. + "com.google.protobuf:protobuf-java:$protobuf_version", + "com.google.protobuf:protobuf-java-util:$protobuf_version", ], runtimeDependencies: [ library.java.slf4j_api,