Skip to content

[BEAM-10961] Enable strict dependency checking on sdks/java/extensions/sql #13830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ibzib merged 15 commits into from
Feb 25, 2021
Merged

[BEAM-10961] Enable strict dependency checking on sdks/java/extensions/sql #13830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
e0d609c
initial move of sdks/java/extensions/sql files to their own PR
shehzaadn-vd Jan 28, 2021
2c1b817
enableStrictDependencies: true
shehzaadn-vd Jan 28, 2021
d37416c
remove unnecessary declaration of checkerqual
shehzaadn-vd Jan 28, 2021
d97ba09
use of existing variables and alphabetacization
shehzaadn-vd Jan 28, 2021
dba8ed6
Incorporate review comments (various)
shehzaadn-vd Feb 1, 2021
a449640
merged from master and resolved conflict in java/extensions/sql/zetas…
shehzaadn-vd Feb 6, 2021
8381744
[BEAM-10961] add compile lines due to new udf dependency
shehzaadn-vd Feb 6, 2021
4df10de
Merge branch 'master' into sdks-extensions-sql
shehzaadn-vd Feb 9, 2021
fc604e4
[BEAM-10961] add permitUnusedDeclared to handle BEAM-11761 and some a…
shehzaadn-vd Feb 10, 2021
48b6def
merged from upstream/master
shehzaadn-vd Feb 12, 2021
994d63b
[BEAM-10961] account for upgraded versions of com.google.api.grpc:*-g…
shehzaadn-vd Feb 12, 2021
621310f
[BEAM-10961] cleanup due to last review comments
shehzaadn-vd Feb 13, 2021
3dc7800
[BEAM-10961] use enforcedPlatform dependency to allow permitUnusedDec…
shehzaadn-vd Feb 18, 2021
5eb8f52
merge master
shehzaadn-vd Feb 24, 2021
3835496
[BEAM-10961] add reference to BEAM-11761
shehzaadn-vd Feb 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,7 @@ class BeamModulePlugin implements Plugin<Project> {
grpc_core : "io.grpc:grpc-core", // google_cloud_platform_libraries_bom sets version
grpc_google_cloud_pubsub_v1 : "com.google.api.grpc:grpc-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version
grpc_google_cloud_pubsublite_v1 : "com.google.api.grpc:grpc-google-cloud-pubsublite-v1:$google_cloud_pubsublite_version",
grpc_google_common_protos : "com.google.api.grpc:grpc-google-common-protos", // google_cloud_platform_libraries_bom sets version
grpc_grpclb : "io.grpc:grpc-grpclb", // google_cloud_platform_libraries_bom sets version
grpc_protobuf : "io.grpc:grpc-protobuf", // google_cloud_platform_libraries_bom sets version
grpc_protobuf_lite : "io.grpc:grpc-protobuf-lite:$grpc_version",
Expand Down Expand Up @@ -602,6 +603,7 @@ class BeamModulePlugin implements Plugin<Project> {
protobuf_java_util : "com.google.protobuf:protobuf-java-util:$protobuf_version",
proto_google_cloud_bigquerybeta2_storage_v1 : "com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2", // google_cloud_platform_libraries_bom sets version
proto_google_cloud_bigtable_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-v2", // google_cloud_platform_libraries_bom sets version
proto_google_cloud_datacatalog_v1beta1 : "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1", // google_cloud_platform_libraries_bom sets version
proto_google_cloud_datastore_v1 : "com.google.api.grpc:proto-google-cloud-datastore-v1", // google_cloud_platform_libraries_bom sets version
proto_google_cloud_pubsub_v1 : "com.google.api.grpc:proto-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version
proto_google_cloud_pubsublite_v1 : "com.google.api.grpc:proto-google-cloud-pubsublite-v1:$google_cloud_pubsublite_version",
Expand Down
13 changes: 8 additions & 5 deletions sdks/java/extensions/sql/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ plugins {
id 'ca.coglinc.javacc'
}
applyJavaNature(
enableStrictDependencies: true,
generatedClassPatterns: [
/^org\.apache\.beam\.sdk\.extensions\.sql\.impl\.parser\.impl.*/,
],
Expand Down Expand Up @@ -57,8 +58,9 @@ dependencies {
fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
fmppTask "org.freemarker:freemarker:2.3.28"
fmppTemplates library.java.vendored_calcite_1_20_0
compile project(":sdks:java:core")
compile project(path: ":sdks:java:core", configuration: "shadow")
compile project(":sdks:java:extensions:join-library")
permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761
compile project(":sdks:java:extensions:sql:udf")
compile project(path: ":runners:direct-java", configuration: "shadow")
compile library.java.commons_codec
Expand All @@ -69,7 +71,11 @@ dependencies {
compile "com.alibaba:fastjson:1.2.69"
compile "org.codehaus.janino:janino:3.0.11"
compile "org.codehaus.janino:commons-compiler:3.0.11"
provided "org.checkerframework:checker-qual:3.10.0"
compile library.java.jackson_core
compile library.java.mongo_java_driver
compile library.java.slf4j_api
compile library.java.joda_time
compile library.java.vendored_guava_26_0_jre
provided project(":sdks:java:io:kafka")
provided project(":sdks:java:io:google-cloud-platform")
compile project(":sdks:java:io:mongodb")
Expand All @@ -80,9 +86,6 @@ dependencies {
testCompile library.java.vendored_calcite_1_20_0
testCompile library.java.vendored_guava_26_0_jre
testCompile library.java.junit
testCompile library.java.hamcrest_core
testCompile library.java.hamcrest_library
testCompile library.java.mockito_core
testCompile library.java.quickcheck_core
testCompile library.java.testcontainers_kafka
testCompile library.java.google_cloud_bigtable_emulator
Expand Down
12 changes: 11 additions & 1 deletion sdks/java/extensions/sql/datacatalog/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,23 @@ import groovy.json.JsonOutput

plugins { id 'org.apache.beam.module' }

applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.datacatalog')
applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.datacatalog')

dependencies {
compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
compile(library.java.google_cloud_datacatalog_v1beta1) {
exclude group: 'io.grpc', module: 'grpc-core' // Use Beam's version
}
compile library.java.gax
compile library.java.google_auth_library_credentials
compile library.java.proto_google_cloud_datacatalog_v1beta1
compile library.java.protobuf_java
compile library.java.slf4j_api
compile library.java.vendored_guava_26_0_jre
compile project(path: ":sdks:java:core", configuration: "shadow")
compile "org.threeten:threetenbp:1.4.5"
provided project(":sdks:java:extensions:sql")

// Dependencies for the example
Expand Down
6 changes: 5 additions & 1 deletion sdks/java/extensions/sql/expansion-service/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
plugins { id 'org.apache.beam.module' }

applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdks.extensions.sql.expansion',
validateShadowJar: false,
shadowClosure: {
Expand All @@ -32,9 +33,12 @@ ext.summary = """Contains code to run a SQL Expansion Service."""


dependencies {
compile project(path: ":sdks:java:core", configuration: "shadow")
compile project(path: ":sdks:java:expansion-service")
permitUnusedDeclared project(path: ":sdks:java:expansion-service") // BEAM-11761
compile project(path: ":sdks:java:extensions:sql")
compile project(path: ":sdks:java:extensions:sql:zetasql")
compile project(path: ":sdks:java:expansion-service")
compile library.java.vendored_guava_26_0_jre
}

task runExpansionService (type: JavaExec) {
Expand Down
1 change: 1 addition & 0 deletions sdks/java/extensions/sql/hcatalog/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
plugins { id 'org.apache.beam.module' }

applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog',
classesTriggerCheckerBugs: [
'HCatalogTable': 'TODO: file a bug report',
Expand Down
7 changes: 4 additions & 3 deletions sdks/java/extensions/sql/jdbc/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import groovy.json.JsonOutput

plugins { id 'org.apache.beam.module' }
applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.jdbc',
enableStrictDependencies: true,
exportJavadoc: false,
testShadowJar: true,
validateShadowJar: false,
Expand All @@ -33,17 +34,17 @@ configurations {
dependencies {
compile project(":sdks:java:extensions:sql")
compile "jline:jline:2.14.6"
permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761
compile "sqlline:sqlline:1.4.0"
compile library.java.slf4j_jdk14
compile library.java.vendored_guava_26_0_jre
testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime")
testCompile library.java.junit
testCompile library.java.hamcrest_core
testCompile library.java.hamcrest_library
// Depending on outputs so integrationTest can run with only test dependencies.
// This enables us to test the JDBC jar being loaded on a custom classloader.
integrationTest sourceSets.test.output
integrationTest sourceSets.main.output
integrationTest library.java.junit
compile project(path: ":sdks:java:core", configuration: "shadow")
}

processResources {
Expand Down
4 changes: 3 additions & 1 deletion sdks/java/extensions/sql/perf-tests/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
*/

plugins { id 'org.apache.beam.module' }
applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider')
applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider')
provideIntegrationTestingDependencies()
enableJavaPerformanceTesting()

Expand Down
1 change: 1 addition & 0 deletions sdks/java/extensions/sql/udf-test-provider/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ plugins {
}

applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.provider',
publish: false,
)
Expand Down
4 changes: 3 additions & 1 deletion sdks/java/extensions/sql/udf/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@

plugins { id 'org.apache.beam.module' }

applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.udf')
applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.udf')

description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL :: UDF"
ext.summary = "Provides interfaces for defining user-defined functions in Beam SQL."
Expand Down
20 changes: 14 additions & 6 deletions sdks/java/extensions/sql/zetasql/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ plugins {
}

applyJavaNature(
enableStrictDependencies: true,
automaticModuleName: 'org.apache.beam.sdk.extensions.sql.zetasql',
)

Expand All @@ -31,23 +32,30 @@ def zetasql_version = "2020.10.1"

dependencies {
compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
compile project(":sdks:java:core")
permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
compile project(path: ":sdks:java:core", configuration: "shadow")
compile project(":sdks:java:extensions:sql")
compile project(":sdks:java:extensions:sql:udf")
compile library.java.vendored_calcite_1_20_0
compile library.java.guava
compile library.java.grpc_api
compile library.java.joda_time
compile library.java.protobuf_java
compile library.java.protobuf_java_util
compile "com.google.api.grpc:proto-google-common-protos:1.12.0" // Interfaces with ZetaSQL use this
compile "com.google.api.grpc:grpc-google-common-protos:1.12.0" // Interfaces with ZetaSQL use this
compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version"
permitUnusedDeclared library.java.protobuf_java_util // BEAM-11761
compile library.java.slf4j_api
compile library.java.vendored_guava_26_0_jre
compile library.java.proto_google_common_protos // Interfaces with ZetaSQL use this
permitUnusedDeclared library.java.proto_google_common_protos // BEAM-11761
compile library.java.grpc_google_common_protos // Interfaces with ZetaSQL use this
permitUnusedDeclared library.java.grpc_google_common_protos // BEAM-11761
compile "com.google.zetasql:zetasql-client:$zetasql_version"
compile "com.google.zetasql:zetasql-types:$zetasql_version"
compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version"
permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" // BEAM-11761
testCompile library.java.vendored_calcite_1_20_0
testCompile library.java.vendored_guava_26_0_jre
testCompile library.java.junit
testCompile library.java.hamcrest_core
testCompile library.java.hamcrest_library
testCompile library.java.mockito_core
testCompile library.java.quickcheck_core
testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider")
Expand Down