diff --git a/.test-infra/validate-runner/build.gradle b/.test-infra/validate-runner/build.gradle index 1d5f389d5ae8..ded6572e3f99 100644 --- a/.test-infra/validate-runner/build.gradle +++ b/.test-infra/validate-runner/build.gradle @@ -22,6 +22,7 @@ group 'org.apache.beam' description = "Apache Beam :: Validate :: Runner" repositories { + maven { url "${project.rootDir}/tempLib" } mavenCentral() maven { url "https://repo.jenkins-ci.org/releases/" diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 55116f855d88..dbf090e05476 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -679,7 +679,7 @@ class BeamModulePlugin implements Plugin { vendored_bytebuddy_1_11_0 : "org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1", vendored_grpc_1_36_0 : "org.apache.beam:beam-vendor-grpc-1_36_0:0.2", vendored_guava_26_0_jre : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1", - vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.1", + vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2-SNAPSHOT", woodstox_core_asl : "org.codehaus.woodstox:woodstox-core-asl:4.4.1", zstd_jni : "com.github.luben:zstd-jni:1.4.5-2", quickcheck_core : "com.pholser:junit-quickcheck-core:$quickcheck_version", diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy index ad4099fbaddd..fbfa1f367665 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy @@ -25,6 +25,7 @@ class Repositories { static void register(Project project) { project.repositories { + maven { url "${project.rootDir}/tempLib" } maven { url project.offlineRepositoryRoot } // To run gradle in offline mode, one must first invoke diff --git a/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.jar b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.jar new file mode 100644 index 000000000000..43d06510c249 Binary files /dev/null and b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.jar differ diff --git a/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.pom b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.pom new file mode 100644 index 000000000000..5370c9c6b7d0 --- /dev/null +++ b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/beam-vendor-calcite-1_28_0-0.2-SNAPSHOT.pom @@ -0,0 +1,80 @@ + + + + 4.0.0 + org.apache.beam + beam-vendor-calcite-1_28_0 + 0.2-SNAPSHOT + Apache Beam :: Vendored Dependencies :: Calcite 1.28.0 + http://beam.apache.org + 2016 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + The Apache Beam Team + dev@beam.apache.org + http://beam.apache.org + Apache Software Foundation + http://www.apache.org + + + + + Beam Dev + dev-subscribe@beam.apache.org + dev-unsubscribe@beam.apache.org + dev@beam.apache.org + http://www.mail-archive.com/dev%beam.apache.org + + + Beam User + user-subscribe@beam.apache.org + user-unsubscribe@beam.apache.org + user@beam.apache.org + http://www.mail-archive.com/user%beam.apache.org + + + Beam Commits + commits-subscribe@beam.apache.org + commits-unsubscribe@beam.apache.org + commits@beam.apache.org + http://www.mail-archive.com/commits%beam.apache.org + + + + scm:git:https://gitbox.apache.org/repos/asf/beam.git + scm:git:https://gitbox.apache.org/repos/asf/beam.git + https://gitbox.apache.org/repos/asf?p=beam.git;a=summary + + + jira + https://issues.apache.org/jira/browse/BEAM + + + + org.slf4j + slf4j-api + 1.7.30 + runtime + + + diff --git a/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/maven-metadata-local.xml b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/maven-metadata-local.xml new file mode 100644 index 000000000000..2137a5c5f3ce --- /dev/null +++ b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/0.2-SNAPSHOT/maven-metadata-local.xml @@ -0,0 +1,24 @@ + + + org.apache.beam + beam-vendor-calcite-1_28_0 + 0.2-SNAPSHOT + + + true + + 20220111012307 + + + jar + 0.2-SNAPSHOT + 20220111012307 + + + pom + 0.2-SNAPSHOT + 20220111012307 + + + + diff --git a/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/maven-metadata-local.xml b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/maven-metadata-local.xml new file mode 100644 index 000000000000..a0dd0c09f001 --- /dev/null +++ b/tempLib/org/apache/beam/beam-vendor-calcite-1_28_0/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + org.apache.beam + beam-vendor-calcite-1_28_0 + + 0.2-SNAPSHOT + + 0.2-SNAPSHOT + + 20220111012307 + + diff --git a/vendor/README.md b/vendor/README.md index a552494bb300..e5eddba98b7d 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -28,6 +28,7 @@ The upgrading of the vendored dependencies should be performed in two steps: # How to validate the vendored dependencies +## Linkage Tool The [linkage tool](https://lists.apache.org/thread.html/eb5d95b9a33d7e32dc9bcd0f7d48ba8711d42bd7ed03b9cf0f1103f1%40%3Cdev.beam.apache.org%3E) is useful for the vendored dependency upgrades. It reports the linkage errors across multiple Apache Beam artifact ids. @@ -48,7 +49,7 @@ $ mvn install:install-file \ $ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_36_0:0.1 :checkJavaLinkage ``` -## Known Linkage Errors in the Vendored gRPC Dependencies +### Known Linkage Errors in the Vendored gRPC Dependencies It's expected that the task outputs some linkage errors. While the `checkJavaLinkage` task does not retrieve optional dependencies to avoid bloated @@ -78,3 +79,38 @@ references to the missing classes. Here are the known linkage errors: to be included in the vendored artifact. Slf4j-api is available at Beam's runtime. - References to `reactor.blockhound`: When enabled, Netty's BlockHound integration can detect unexpected blocking calls. Beam does not use it. + +## Create testing PR against new artifacts + +Once you've verified using the linkage tool, you can test new artifacts by running unit and integration tests against a PR. + +Example PRs: +- Updating gRPC version (large) https://github.com/apache/beam/pull/16460 +- Updating protobuf for calcite (minor version update): https://github.com/apache/beam/pull/16476 + +Steps: + +1. Generate new artifact files with `publishMavenJavaPublicationToMavenLocal`, e.g. + +``` +./gradlew -p vendor/grpc-1_43_2 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly + +# Copy files (jar/poms/metadata) to your beam repository +cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_43_2/ \ + $BEAMDIR/tempLib/org/apache/beam/beam-vendor-grpc-1_43_2 +``` + +2. Add whatever folder (here I use `tempLib`) to the expected project repositories, e.g. + +``` +repositories { + maven { url "${project.rootDir}/tempLib" } + maven { + ... + } +} +``` + +3. Migrate all references from the old dependency to the new dependency, including imports if needed. + +4. Commit any added or changed files and create a PR (can be a draft, as you will not merge this PR) to test on. diff --git a/vendor/calcite-1_28_0/build.gradle b/vendor/calcite-1_28_0/build.gradle index adf154bc6fd5..e2e75b45aebf 100644 --- a/vendor/calcite-1_28_0/build.gradle +++ b/vendor/calcite-1_28_0/build.gradle @@ -16,21 +16,35 @@ * limitations under the License. */ +/** + * Vendored version of calcite. + * + * To upgrade: + * 1. Use mvn dependency:tree and/or https://search.maven.org/search?q=g:org.apache.calcite%20AND%20a:calcite-core + * to determine dependency tree. You may need to search for optional transitive dependencies + * and determine if they need to be added or upgraded (e.g. protobuf) + * 3. Validate built artifacts by running linkage tool + * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) + * and unit and integration tests in a PR. + */ + plugins { id 'org.apache.beam.vendor-java' } description = "Apache Beam :: Vendored Dependencies :: Calcite 1.28.0" group = "org.apache.beam" -version = "0.1" +version = "0.2" def calcite_version = "1.28.0" def avatica_version = "1.19.0" +def protobuf_version = "3.19.2" def prefix = "org.apache.beam.vendor.calcite.v1_28_0" List packagesToRelocate = [ "com.esri", "com.fasterxml", "com.google.common", + "com.google.gson", "com.google.protobuf", "com.google.thirdparty", "com.google.uzaygezen", @@ -52,6 +66,12 @@ vendorJava( "org.apache.calcite:calcite-core:$calcite_version", "org.apache.calcite:calcite-linq4j:$calcite_version", "org.apache.calcite.avatica:avatica-core:$avatica_version", + + // BEAM-13616: Override the version of protobuf to patch a security vulnerability. + // This override can be removed once we upgrade to a newer version of calcite that + // depends on protobuf >= 3.19.2. + "com.google.protobuf:protobuf-java:$protobuf_version", + "com.google.protobuf:protobuf-java-util:$protobuf_version", ], runtimeDependencies: [ library.java.slf4j_api,