From e0d609cdaf97387ee8280fffb4026b1600c3b627 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 27 Jan 2021 20:47:18 -0800 Subject: [PATCH 01/11] initial move of sdks/java/extensions/sql files to their own PR --- sdks/java/extensions/sql/build.gradle | 13 ++++++++----- sdks/java/extensions/sql/datacatalog/build.gradle | 11 +++++++++++ .../extensions/sql/expansion-service/build.gradle | 4 +++- sdks/java/extensions/sql/hcatalog/build.gradle | 1 - sdks/java/extensions/sql/jdbc/build.gradle | 7 +++---- sdks/java/extensions/sql/zetasql/build.gradle | 15 +++++++-------- 6 files changed, 32 insertions(+), 19 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 93473852ef81..53c3aa31e146 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -62,7 +62,6 @@ dependencies { fmppTask "org.freemarker:freemarker:2.3.28" fmppTemplates library.java.vendored_calcite_1_20_0 compile project(":sdks:java:core") - compile project(":sdks:java:extensions:join-library") compile project(":sdks:java:extensions:sql:udf") compile project(path: ":runners:direct-java", configuration: "shadow") compile library.java.commons_codec @@ -73,7 +72,14 @@ dependencies { compile "com.alibaba:fastjson:1.2.69" compile "org.codehaus.janino:janino:3.0.11" compile "org.codehaus.janino:commons-compiler:3.0.11" - provided "org.checkerframework:checker-qual:3.4.1" + compile library.java.jackson_core + compile library.java.vendored_guava_26_0_jre + compile "org.mongodb:mongo-java-driver:3.9.1" + compile library.java.slf4j_api + compile library.java.joda_time + compile library.java.vendored_guava_26_0_jre + compile "org.checkerframework:checker-qual:2.0.0" + permitUnusedDeclared project(":sdks:java:core") provided project(":sdks:java:io:kafka") provided project(":sdks:java:io:google-cloud-platform") compile project(":sdks:java:io:mongodb") @@ -85,9 +91,6 @@ dependencies { testCompile library.java.vendored_calcite_1_20_0 testCompile library.java.vendored_guava_26_0_jre testCompile library.java.junit - testCompile library.java.hamcrest_core - testCompile library.java.hamcrest_library - testCompile library.java.mockito_core testCompile library.java.quickcheck_core testCompile library.java.testcontainers_kafka testCompile library.java.google_cloud_bigtable_emulator diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index ad1fab0bceb9..c551c5dac582 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -27,6 +27,15 @@ dependencies { compile(library.java.google_cloud_datacatalog_v1beta1) { exclude group: 'io.grpc', module: 'grpc-core' // Use Beam's version } + compile "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1:0.32.1" + compile "com.google.api:gax:1.57.1" + compile "com.google.auth:google-auth-library-credentials:0.19.0" + compile "org.checkerframework:checker-qual:2.0.0" + compile library.java.vendored_guava_26_0_jre + compile library.java.slf4j_api + compile project(path: ":sdks:java:core", configuration: "shadow") + compile "org.threeten:threetenbp:1.4.5" + compile library.java.protobuf_java provided project(":sdks:java:extensions:sql") // Dependencies for the example @@ -35,6 +44,8 @@ dependencies { testCompile project(":sdks:java:extensions:sql:zetasql") testRuntimeOnly library.java.slf4j_simple + permitUnusedDeclared "org.threeten:threetenbp:1.4.5" + permitUnusedDeclared library.java.protobuf_java } task runDataCatalogExample(type: JavaExec) { diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index aabd3b1ed0c2..2578510cc689 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -34,7 +34,9 @@ ext.summary = """Contains code to run a SQL Expansion Service.""" dependencies { compile project(path: ":sdks:java:extensions:sql") compile project(path: ":sdks:java:extensions:sql:zetasql") - compile project(path: ":sdks:java:expansion-service") + compile project(path: ":sdks:java:core", configuration: "shadow") + compile "org.apache.beam:beam-vendor-guava-26_0-jre:0.1" + compile "org.checkerframework:checker-qual:2.0.0" } task runExpansionService (type: JavaExec) { diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 678492eed4c9..81bca1a62f97 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -38,6 +38,5 @@ dependencies { testCompile("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") { // Hive brings full Calcite 1.6 + Avatica with JDBC driver which // gets registered and gets started instead of ours - exclude group: "org.apache.calcite", module:"calcite-avatica" } } diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index f1919053a9e7..f051f5a3ebf3 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -32,18 +32,17 @@ configurations { dependencies { compile project(":sdks:java:extensions:sql") - compile "jline:jline:2.14.6" compile "sqlline:sqlline:1.4.0" - compile library.java.slf4j_jdk14 + compile library.java.vendored_guava_26_0_jre testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime") testCompile library.java.junit - testCompile library.java.hamcrest_core - testCompile library.java.hamcrest_library // Depending on outputs so integrationTest can run with only test dependencies. // This enables us to test the JDBC jar being loaded on a custom classloader. integrationTest sourceSets.test.output integrationTest sourceSets.main.output integrationTest library.java.junit + compile project(path: ":sdks:java:core", configuration: "shadow") + compile "org.checkerframework:checker-qual:2.0.0" } processResources { diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 9589301e90ca..e486bc805dac 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -31,25 +31,24 @@ def zetasql_version = "2020.10.1" dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) - compile project(":sdks:java:core") + compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:sql") compile library.java.vendored_calcite_1_20_0 compile library.java.guava compile library.java.grpc_api compile library.java.protobuf_java - compile library.java.protobuf_java_util - compile "com.google.api.grpc:proto-google-common-protos:1.12.0" // Interfaces with ZetaSQL use this - compile "com.google.api.grpc:grpc-google-common-protos:1.12.0" // Interfaces with ZetaSQL use this - compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" compile "com.google.zetasql:zetasql-client:$zetasql_version" compile "com.google.zetasql:zetasql-types:$zetasql_version" + compile library.java.joda_time + compile library.java.vendored_guava_26_0_jre + compile "org.checkerframework:checker-qual:2.0.0" + permitUnusedDeclared library.java.protobuf_java_util + compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" + permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" testCompile library.java.vendored_calcite_1_20_0 testCompile library.java.vendored_guava_26_0_jre testCompile library.java.junit - testCompile library.java.hamcrest_core - testCompile library.java.hamcrest_library testCompile library.java.mockito_core - testCompile library.java.quickcheck_core testRuntimeClasspath library.java.slf4j_jdk14 } From 2c1b8173df535170cd8d012153c3d13d12b0909a Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 27 Jan 2021 20:54:41 -0800 Subject: [PATCH 02/11] enableStrictDependencies: true --- sdks/java/extensions/sql/build.gradle | 1 + sdks/java/extensions/sql/datacatalog/build.gradle | 4 +++- sdks/java/extensions/sql/expansion-service/build.gradle | 1 + sdks/java/extensions/sql/hcatalog/build.gradle | 1 + sdks/java/extensions/sql/jdbc/build.gradle | 1 + sdks/java/extensions/sql/perf-tests/build.gradle | 4 +++- sdks/java/extensions/sql/udf-test-provider/build.gradle | 4 +++- sdks/java/extensions/sql/udf/build.gradle | 4 +++- sdks/java/extensions/sql/zetasql/build.gradle | 1 + 9 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 53c3aa31e146..4c1de6bddd42 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -24,6 +24,7 @@ plugins { id 'ca.coglinc.javacc' } applyJavaNature( + enableStrictDependencies: true, generatedClassPatterns: [ /^org\.apache\.beam\.sdk\.extensions\.sql\.impl\.parser\.impl.*/, /^org\.apache\.beam\.sdk\.extensions\.sql\.meta\.provider\.kafka\.KafkaMessages/, diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index c551c5dac582..73546bdfea10 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -20,7 +20,9 @@ import groovy.json.JsonOutput plugins { id 'org.apache.beam.module' } -applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.datacatalog') +applyJavaNature( + enableStrictDependencies: true, + automaticModuleName: 'org.apache.beam.sdk.extensions.sql.datacatalog') dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index 2578510cc689..cc3e4594c704 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -18,6 +18,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( + enableStrictDependencies: true, automaticModuleName: 'org.apache.beam.sdks.extensions.sql.expansion', validateShadowJar: false, shadowClosure: { diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 81bca1a62f97..2ae62e6993d0 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -19,6 +19,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( + enableStrictDependencies: true, automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog', classesTriggerCheckerBugs: [ 'HCatalogTable': 'TODO: file a bug report', diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index f051f5a3ebf3..93714ea6ed73 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -20,6 +20,7 @@ import groovy.json.JsonOutput plugins { id 'org.apache.beam.module' } applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.jdbc', + enableStrictDependencies: true, exportJavadoc: false, testShadowJar: true, validateShadowJar: false, diff --git a/sdks/java/extensions/sql/perf-tests/build.gradle b/sdks/java/extensions/sql/perf-tests/build.gradle index 7875a6b846fd..4c24bce4ff60 100644 --- a/sdks/java/extensions/sql/perf-tests/build.gradle +++ b/sdks/java/extensions/sql/perf-tests/build.gradle @@ -17,7 +17,9 @@ */ plugins { id 'org.apache.beam.module' } -applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider') +applyJavaNature( + enableStrictDependencies: true, + automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider') provideIntegrationTestingDependencies() enableJavaPerformanceTesting() diff --git a/sdks/java/extensions/sql/udf-test-provider/build.gradle b/sdks/java/extensions/sql/udf-test-provider/build.gradle index 6769eba1d994..314dcfa47da1 100644 --- a/sdks/java/extensions/sql/udf-test-provider/build.gradle +++ b/sdks/java/extensions/sql/udf-test-provider/build.gradle @@ -20,7 +20,9 @@ plugins { id 'org.apache.beam.module' } -applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.provider') +applyJavaNature( + enableStrictDependencies: true, + automaticModuleName: 'org.apache.beam.sdk.extensions.sql.provider') description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL :: UDF test provider" ext.summary = "Java UDFs for testing. This project must be built separately from its parent so the UDF provider is not included in the context classloader for tests." diff --git a/sdks/java/extensions/sql/udf/build.gradle b/sdks/java/extensions/sql/udf/build.gradle index 4fc392e06ff2..adb6e7c83749 100644 --- a/sdks/java/extensions/sql/udf/build.gradle +++ b/sdks/java/extensions/sql/udf/build.gradle @@ -18,7 +18,9 @@ plugins { id 'org.apache.beam.module' } -applyJavaNature(automaticModuleName: 'org.apache.beam.sdk.extensions.sql.udf') +applyJavaNature( + enableStrictDependencies: true, + automaticModuleName: 'org.apache.beam.sdk.extensions.sql.udf') description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL :: UDF" ext.summary = "Provides interfaces for defining user-defined functions in Beam SQL." diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index e486bc805dac..398f20dd3a8c 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -21,6 +21,7 @@ plugins { } applyJavaNature( + enableStrictDependencies: true, automaticModuleName: 'org.apache.beam.sdk.extensions.sql.zetasql', ) From d37416cac45e8588f6a3157d7a250c0d71f35b35 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 27 Jan 2021 21:03:20 -0800 Subject: [PATCH 03/11] remove unnecessary declaration of checkerqual --- sdks/java/extensions/sql/build.gradle | 1 - sdks/java/extensions/sql/datacatalog/build.gradle | 1 - sdks/java/extensions/sql/expansion-service/build.gradle | 1 - sdks/java/extensions/sql/jdbc/build.gradle | 1 - sdks/java/extensions/sql/zetasql/build.gradle | 1 - 5 files changed, 5 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 4c1de6bddd42..f72e8007793d 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -79,7 +79,6 @@ dependencies { compile library.java.slf4j_api compile library.java.joda_time compile library.java.vendored_guava_26_0_jre - compile "org.checkerframework:checker-qual:2.0.0" permitUnusedDeclared project(":sdks:java:core") provided project(":sdks:java:io:kafka") provided project(":sdks:java:io:google-cloud-platform") diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 73546bdfea10..0d648ff3719d 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -32,7 +32,6 @@ dependencies { compile "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1:0.32.1" compile "com.google.api:gax:1.57.1" compile "com.google.auth:google-auth-library-credentials:0.19.0" - compile "org.checkerframework:checker-qual:2.0.0" compile library.java.vendored_guava_26_0_jre compile library.java.slf4j_api compile project(path: ":sdks:java:core", configuration: "shadow") diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index cc3e4594c704..79d4ee34257e 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -37,7 +37,6 @@ dependencies { compile project(path: ":sdks:java:extensions:sql:zetasql") compile project(path: ":sdks:java:core", configuration: "shadow") compile "org.apache.beam:beam-vendor-guava-26_0-jre:0.1" - compile "org.checkerframework:checker-qual:2.0.0" } task runExpansionService (type: JavaExec) { diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 93714ea6ed73..cb170608305a 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -43,7 +43,6 @@ dependencies { integrationTest sourceSets.main.output integrationTest library.java.junit compile project(path: ":sdks:java:core", configuration: "shadow") - compile "org.checkerframework:checker-qual:2.0.0" } processResources { diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 398f20dd3a8c..93960431858b 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -42,7 +42,6 @@ dependencies { compile "com.google.zetasql:zetasql-types:$zetasql_version" compile library.java.joda_time compile library.java.vendored_guava_26_0_jre - compile "org.checkerframework:checker-qual:2.0.0" permitUnusedDeclared library.java.protobuf_java_util compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" From d97ba0999f1cb4230b0c18cc740eb76a62caf3d1 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 27 Jan 2021 21:11:21 -0800 Subject: [PATCH 04/11] use of existing variables and alphabetacization --- sdks/java/extensions/sql/datacatalog/build.gradle | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 0d648ff3719d..88cdbe388349 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -29,14 +29,14 @@ dependencies { compile(library.java.google_cloud_datacatalog_v1beta1) { exclude group: 'io.grpc', module: 'grpc-core' // Use Beam's version } - compile "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1:0.32.1" - compile "com.google.api:gax:1.57.1" - compile "com.google.auth:google-auth-library-credentials:0.19.0" - compile library.java.vendored_guava_26_0_jre + compile library.java.gax + compile library.java.google_auth_library_credentials + compile library.java.protobuf_java compile library.java.slf4j_api + compile library.java.vendored_guava_26_0_jre compile project(path: ":sdks:java:core", configuration: "shadow") + compile "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1:0.32.1" compile "org.threeten:threetenbp:1.4.5" - compile library.java.protobuf_java provided project(":sdks:java:extensions:sql") // Dependencies for the example From dba8ed652af9865ac2ed92e1c665a98c8e6ab44b Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Sun, 31 Jan 2021 18:32:34 -0800 Subject: [PATCH 05/11] Incorporate review comments (various) --- .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + sdks/java/extensions/sql/build.gradle | 2 +- sdks/java/extensions/sql/datacatalog/build.gradle | 6 +++--- sdks/java/extensions/sql/expansion-service/build.gradle | 2 +- sdks/java/extensions/sql/hcatalog/build.gradle | 1 + sdks/java/extensions/sql/jdbc/build.gradle | 2 ++ sdks/java/extensions/sql/zetasql/build.gradle | 1 - 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index c70c405ae62f..f8abdffefc06 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -602,6 +602,7 @@ class BeamModulePlugin implements Plugin { protobuf_java_util : "com.google.protobuf:protobuf-java-util:$protobuf_version", proto_google_cloud_bigquery_storage_v1beta1 : "com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1", // google_cloud_platform_libraries_bom sets version proto_google_cloud_bigtable_v2 : "com.google.api.grpc:proto-google-cloud-bigtable-v2", // google_cloud_platform_libraries_bom sets version + proto_google_cloud_datacatalog_v1beta1 : "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1", // google_cloud_platform_libraries_bom sets version proto_google_cloud_datastore_v1 : "com.google.api.grpc:proto-google-cloud-datastore-v1", // google_cloud_platform_libraries_bom sets version proto_google_cloud_pubsub_v1 : "com.google.api.grpc:proto-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version proto_google_cloud_pubsublite_v1 : "com.google.api.grpc:proto-google-cloud-pubsublite-v1:$google_cloud_pubsublite_version", diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index f72e8007793d..0090a1241cbb 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -62,7 +62,7 @@ dependencies { fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0" fmppTask "org.freemarker:freemarker:2.3.28" fmppTemplates library.java.vendored_calcite_1_20_0 - compile project(":sdks:java:core") + compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:sql:udf") compile project(path: ":runners:direct-java", configuration: "shadow") compile library.java.commons_codec diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 88cdbe388349..422eac8f836f 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -31,12 +31,14 @@ dependencies { } compile library.java.gax compile library.java.google_auth_library_credentials + compile library.java.proto_google_cloud_datacatalog_v1beta1 compile library.java.protobuf_java + permitUnusedDeclared library.java.protobuf_java compile library.java.slf4j_api compile library.java.vendored_guava_26_0_jre compile project(path: ":sdks:java:core", configuration: "shadow") - compile "com.google.api.grpc:proto-google-cloud-datacatalog-v1beta1:0.32.1" compile "org.threeten:threetenbp:1.4.5" + permitUnusedDeclared "org.threeten:threetenbp:1.4.5" provided project(":sdks:java:extensions:sql") // Dependencies for the example @@ -45,8 +47,6 @@ dependencies { testCompile project(":sdks:java:extensions:sql:zetasql") testRuntimeOnly library.java.slf4j_simple - permitUnusedDeclared "org.threeten:threetenbp:1.4.5" - permitUnusedDeclared library.java.protobuf_java } task runDataCatalogExample(type: JavaExec) { diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index 79d4ee34257e..3e520a04bc46 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -36,7 +36,7 @@ dependencies { compile project(path: ":sdks:java:extensions:sql") compile project(path: ":sdks:java:extensions:sql:zetasql") compile project(path: ":sdks:java:core", configuration: "shadow") - compile "org.apache.beam:beam-vendor-guava-26_0-jre:0.1" + compile library.java.vendored_guava_26_0_jre } task runExpansionService (type: JavaExec) { diff --git a/sdks/java/extensions/sql/hcatalog/build.gradle b/sdks/java/extensions/sql/hcatalog/build.gradle index 2ae62e6993d0..f55391603a3d 100644 --- a/sdks/java/extensions/sql/hcatalog/build.gradle +++ b/sdks/java/extensions/sql/hcatalog/build.gradle @@ -39,5 +39,6 @@ dependencies { testCompile("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") { // Hive brings full Calcite 1.6 + Avatica with JDBC driver which // gets registered and gets started instead of ours + exclude group: "org.apache.calcite", module:"calcite-avatica" } } diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index cb170608305a..818afd835a9b 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -35,6 +35,8 @@ dependencies { compile project(":sdks:java:extensions:sql") compile "sqlline:sqlline:1.4.0" compile library.java.vendored_guava_26_0_jre + runtimeOnly library.java.slf4j_jdk14 + runtimeOnly "jline:jline:2.14.6" testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime") testCompile library.java.junit // Depending on outputs so integrationTest can run with only test dependencies. diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 93960431858b..cb4f192cb85b 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -42,7 +42,6 @@ dependencies { compile "com.google.zetasql:zetasql-types:$zetasql_version" compile library.java.joda_time compile library.java.vendored_guava_26_0_jre - permitUnusedDeclared library.java.protobuf_java_util compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" testCompile library.java.vendored_calcite_1_20_0 From 83817448506bfbfc078700b21bebddf6ee329a02 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Sat, 6 Feb 2021 04:52:20 -0800 Subject: [PATCH 06/11] [BEAM-10961] add compile lines due to new udf dependency --- sdks/java/extensions/sql/zetasql/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 7c6ba098acd7..ca542fbdef61 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -34,10 +34,12 @@ dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:sql") + compile project(":sdks:java:extensions:sql:udf") compile library.java.vendored_calcite_1_20_0 compile library.java.guava compile library.java.grpc_api compile library.java.protobuf_java + compile library.java.slf4j_api compile "com.google.zetasql:zetasql-client:$zetasql_version" compile "com.google.zetasql:zetasql-types:$zetasql_version" compile library.java.joda_time From fc604e4fdc74fe98335790a82d0a9ee2634392fb Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Tue, 9 Feb 2021 22:25:12 -0800 Subject: [PATCH 07/11] [BEAM-10961] add permitUnusedDeclared to handle BEAM-11761 and some alphabeticization --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + sdks/java/extensions/sql/build.gradle | 5 +++-- .../extensions/sql/expansion-service/build.gradle | 4 +++- sdks/java/extensions/sql/jdbc/build.gradle | 2 ++ sdks/java/extensions/sql/zetasql/build.gradle | 12 +++++++++--- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index dc57e28ddf78..90c73e4bce10 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -552,6 +552,7 @@ class BeamModulePlugin implements Plugin { grpc_core : "io.grpc:grpc-core", // google_cloud_platform_libraries_bom sets version grpc_google_cloud_pubsub_v1 : "com.google.api.grpc:grpc-google-cloud-pubsub-v1", // google_cloud_platform_libraries_bom sets version grpc_google_cloud_pubsublite_v1 : "com.google.api.grpc:grpc-google-cloud-pubsublite-v1:$google_cloud_pubsublite_version", + grpc_google_common_protos : "com.google.api.grpc:grpc-google-common-protos", // google_cloud_platform_libraries_bom sets version grpc_grpclb : "io.grpc:grpc-grpclb", // google_cloud_platform_libraries_bom sets version grpc_protobuf : "io.grpc:grpc-protobuf", // google_cloud_platform_libraries_bom sets version grpc_protobuf_lite : "io.grpc:grpc-protobuf-lite:$grpc_version", diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index 0090a1241cbb..d8480fc49872 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -63,6 +63,8 @@ dependencies { fmppTask "org.freemarker:freemarker:2.3.28" fmppTemplates library.java.vendored_calcite_1_20_0 compile project(path: ":sdks:java:core", configuration: "shadow") + compile project(":sdks:java:extensions:join-library") + permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761 compile project(":sdks:java:extensions:sql:udf") compile project(path: ":runners:direct-java", configuration: "shadow") compile library.java.commons_codec @@ -74,12 +76,11 @@ dependencies { compile "org.codehaus.janino:janino:3.0.11" compile "org.codehaus.janino:commons-compiler:3.0.11" compile library.java.jackson_core + compile library.java.mongo_java_driver compile library.java.vendored_guava_26_0_jre - compile "org.mongodb:mongo-java-driver:3.9.1" compile library.java.slf4j_api compile library.java.joda_time compile library.java.vendored_guava_26_0_jre - permitUnusedDeclared project(":sdks:java:core") provided project(":sdks:java:io:kafka") provided project(":sdks:java:io:google-cloud-platform") compile project(":sdks:java:io:mongodb") diff --git a/sdks/java/extensions/sql/expansion-service/build.gradle b/sdks/java/extensions/sql/expansion-service/build.gradle index 3e520a04bc46..c14e141d13b1 100644 --- a/sdks/java/extensions/sql/expansion-service/build.gradle +++ b/sdks/java/extensions/sql/expansion-service/build.gradle @@ -33,9 +33,11 @@ ext.summary = """Contains code to run a SQL Expansion Service.""" dependencies { + compile project(path: ":sdks:java:core", configuration: "shadow") + compile project(path: ":sdks:java:expansion-service") + permitUnusedDeclared project(path: ":sdks:java:expansion-service") // BEAM-11761 compile project(path: ":sdks:java:extensions:sql") compile project(path: ":sdks:java:extensions:sql:zetasql") - compile project(path: ":sdks:java:core", configuration: "shadow") compile library.java.vendored_guava_26_0_jre } diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 818afd835a9b..435cfb78cefe 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -33,6 +33,8 @@ configurations { dependencies { compile project(":sdks:java:extensions:sql") + compile "jline:jline:2.14.6" + permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761 compile "sqlline:sqlline:1.4.0" compile library.java.vendored_guava_26_0_jre runtimeOnly library.java.slf4j_jdk14 diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index ca542fbdef61..f58cbba4193c 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -38,14 +38,20 @@ dependencies { compile library.java.vendored_calcite_1_20_0 compile library.java.guava compile library.java.grpc_api + compile library.java.joda_time compile library.java.protobuf_java + compile library.java.protobuf_java_util + permitUnusedDeclared library.java.protobuf_java_util compile library.java.slf4j_api + compile library.java.vendored_guava_26_0_jre + compile library.java.proto_google_common_protos // Interfaces with ZetaSQL use this + permitUnusedDeclared "com.google.api.grpc:proto-google-common-protos:1.18.1" // BEAM-11761 + compile library.java.grpc_google_common_protos // Interfaces with ZetaSQL use this + permitUnusedDeclared "com.google.api.grpc:grpc-google-common-protos:1.18.1" // BEAM-11761 compile "com.google.zetasql:zetasql-client:$zetasql_version" compile "com.google.zetasql:zetasql-types:$zetasql_version" - compile library.java.joda_time - compile library.java.vendored_guava_26_0_jre compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" - permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" + permitUnusedDeclared "com.google.zetasql:zetasql-jni-channel:$zetasql_version" // BEAM-11761 testCompile library.java.vendored_calcite_1_20_0 testCompile library.java.vendored_guava_26_0_jre testCompile library.java.junit From 994d63bf22b90f3d6b5b828431d9a7adb616070a Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Fri, 12 Feb 2021 01:16:37 -0800 Subject: [PATCH 08/11] [BEAM-10961] account for upgraded versions of com.google.api.grpc:*-google-common-protos --- sdks/java/extensions/sql/zetasql/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index f58cbba4193c..b832a719456f 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -45,9 +45,9 @@ dependencies { compile library.java.slf4j_api compile library.java.vendored_guava_26_0_jre compile library.java.proto_google_common_protos // Interfaces with ZetaSQL use this - permitUnusedDeclared "com.google.api.grpc:proto-google-common-protos:1.18.1" // BEAM-11761 + permitUnusedDeclared "com.google.api.grpc:proto-google-common-protos:2.0.1" // BEAM-11761 compile library.java.grpc_google_common_protos // Interfaces with ZetaSQL use this - permitUnusedDeclared "com.google.api.grpc:grpc-google-common-protos:1.18.1" // BEAM-11761 + permitUnusedDeclared "com.google.api.grpc:grpc-google-common-protos:2.0.1" // BEAM-11761 compile "com.google.zetasql:zetasql-client:$zetasql_version" compile "com.google.zetasql:zetasql-types:$zetasql_version" compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" @@ -64,6 +64,6 @@ dependencies { test { dependsOn ":sdks:java:extensions:sql:emptyJar" // Pass jars used by Java UDF tests via system properties. - systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath - systemProperty "beam.sql.udf.test.empty_jar_path", project(":sdks:java:extensions:sql").emptyJar.archivePath +// systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath +// systemProperty "beam.sql.udf.test.empty_jar_path", project(":sdks:java:extensions:sql").emptyJar.archivePath } From 621310fb43c05e89e23d0fcee5c28430cce3e1bb Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Fri, 12 Feb 2021 16:56:07 -0800 Subject: [PATCH 09/11] [BEAM-10961] cleanup due to last review comments --- sdks/java/extensions/sql/build.gradle | 1 - sdks/java/extensions/sql/datacatalog/build.gradle | 2 -- sdks/java/extensions/sql/jdbc/build.gradle | 2 -- sdks/java/extensions/sql/zetasql/build.gradle | 4 ++-- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle index d8480fc49872..e2ece470ae02 100644 --- a/sdks/java/extensions/sql/build.gradle +++ b/sdks/java/extensions/sql/build.gradle @@ -77,7 +77,6 @@ dependencies { compile "org.codehaus.janino:commons-compiler:3.0.11" compile library.java.jackson_core compile library.java.mongo_java_driver - compile library.java.vendored_guava_26_0_jre compile library.java.slf4j_api compile library.java.joda_time compile library.java.vendored_guava_26_0_jre diff --git a/sdks/java/extensions/sql/datacatalog/build.gradle b/sdks/java/extensions/sql/datacatalog/build.gradle index 422eac8f836f..8b9289d747fe 100644 --- a/sdks/java/extensions/sql/datacatalog/build.gradle +++ b/sdks/java/extensions/sql/datacatalog/build.gradle @@ -33,12 +33,10 @@ dependencies { compile library.java.google_auth_library_credentials compile library.java.proto_google_cloud_datacatalog_v1beta1 compile library.java.protobuf_java - permitUnusedDeclared library.java.protobuf_java compile library.java.slf4j_api compile library.java.vendored_guava_26_0_jre compile project(path: ":sdks:java:core", configuration: "shadow") compile "org.threeten:threetenbp:1.4.5" - permitUnusedDeclared "org.threeten:threetenbp:1.4.5" provided project(":sdks:java:extensions:sql") // Dependencies for the example diff --git a/sdks/java/extensions/sql/jdbc/build.gradle b/sdks/java/extensions/sql/jdbc/build.gradle index 435cfb78cefe..0b2e42e8fd5c 100644 --- a/sdks/java/extensions/sql/jdbc/build.gradle +++ b/sdks/java/extensions/sql/jdbc/build.gradle @@ -37,8 +37,6 @@ dependencies { permitUnusedDeclared "jline:jline:2.14.6" // BEAM-11761 compile "sqlline:sqlline:1.4.0" compile library.java.vendored_guava_26_0_jre - runtimeOnly library.java.slf4j_jdk14 - runtimeOnly "jline:jline:2.14.6" testCompile project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntime") testCompile library.java.junit // Depending on outputs so integrationTest can run with only test dependencies. diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index b832a719456f..53bd144fcb1c 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -64,6 +64,6 @@ dependencies { test { dependsOn ":sdks:java:extensions:sql:emptyJar" // Pass jars used by Java UDF tests via system properties. -// systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath -// systemProperty "beam.sql.udf.test.empty_jar_path", project(":sdks:java:extensions:sql").emptyJar.archivePath + systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath + systemProperty "beam.sql.udf.test.empty_jar_path", project(":sdks:java:extensions:sql").emptyJar.archivePath } From 3dc78008960541f64d9d66ff65a9f22ac5e20b48 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 17 Feb 2021 19:07:23 -0800 Subject: [PATCH 10/11] [BEAM-10961] use enforcedPlatform dependency to allow permitUnusedDeclared to support library.java.* variables --- sdks/java/extensions/sql/zetasql/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 53bd144fcb1c..9f82ad98cb9b 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -32,6 +32,7 @@ def zetasql_version = "2020.10.1" dependencies { compile enforcedPlatform(library.java.google_cloud_platform_libraries_bom) + permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom) compile project(path: ":sdks:java:core", configuration: "shadow") compile project(":sdks:java:extensions:sql") compile project(":sdks:java:extensions:sql:udf") @@ -45,9 +46,9 @@ dependencies { compile library.java.slf4j_api compile library.java.vendored_guava_26_0_jre compile library.java.proto_google_common_protos // Interfaces with ZetaSQL use this - permitUnusedDeclared "com.google.api.grpc:proto-google-common-protos:2.0.1" // BEAM-11761 + permitUnusedDeclared library.java.proto_google_common_protos // BEAM-11761 compile library.java.grpc_google_common_protos // Interfaces with ZetaSQL use this - permitUnusedDeclared "com.google.api.grpc:grpc-google-common-protos:2.0.1" // BEAM-11761 + permitUnusedDeclared library.java.grpc_google_common_protos // BEAM-11761 compile "com.google.zetasql:zetasql-client:$zetasql_version" compile "com.google.zetasql:zetasql-types:$zetasql_version" compile "com.google.zetasql:zetasql-jni-channel:$zetasql_version" From 38354967fc59312bb76b8b83243e59a8c03666d4 Mon Sep 17 00:00:00 2001 From: Shehzaad Nakhoda Date: Wed, 24 Feb 2021 22:09:03 -0800 Subject: [PATCH 11/11] [BEAM-10961] add reference to BEAM-11761 --- sdks/java/extensions/sql/zetasql/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle index 9f82ad98cb9b..9a623d7dccaa 100644 --- a/sdks/java/extensions/sql/zetasql/build.gradle +++ b/sdks/java/extensions/sql/zetasql/build.gradle @@ -42,7 +42,7 @@ dependencies { compile library.java.joda_time compile library.java.protobuf_java compile library.java.protobuf_java_util - permitUnusedDeclared library.java.protobuf_java_util + permitUnusedDeclared library.java.protobuf_java_util // BEAM-11761 compile library.java.slf4j_api compile library.java.vendored_guava_26_0_jre compile library.java.proto_google_common_protos // Interfaces with ZetaSQL use this