[BEAM-9379] Update calcite to 1.26

.gitignore

@@ -17,6 +17,10 @@ runners/**/vendor/**/*
 **/.gradletasknamecache
 **/generated/*
 
+# Ignore sources generated into the main tree
+**/src/main/generated/**
+**/src/test/generated_tests/**
+
 # Ignore files generated by the Maven build process.
 **/bin/**/*
 **/dependency-reduced-pom.xml

CHANGES.md

@@ -64,6 +64,7 @@
 ## New Features / Improvements
 
 * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)).
+* Upgrade to Calcite 1.26.0 ([BEAM-9379](https://issues.apache.org/jira/browse/BEAM-9379)).
 
 ## Breaking Changes
 
@@ -73,6 +74,7 @@
   * Code depending on beam imports need to include v2 on the module path.
     * Fix by'v2' to the import paths, turning  `.../sdks/go/...` to `.../sdks/v2/go/...`
   * No other code change should be required to use v2.33.0 of the Go SDK.
+* SQL Rows are no longer flattened ([BEAM-5505](https://issues.apache.org/jira/browse/BEAM-5505)).
 
 ## Deprecations
 

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -662,7 +662,7 @@ class BeamModulePlugin implements Plugin<Project> {
         vendored_bytebuddy_1_11_0                   : "org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1",
         vendored_grpc_1_36_0                        : "org.apache.beam:beam-vendor-grpc-1_36_0:0.2",
         vendored_guava_26_0_jre                     : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1",
-        vendored_calcite_1_20_0                     : "org.apache.beam:beam-vendor-calcite-1_20_0:0.1",
+        vendored_calcite_1_26_0                     : "org.apache.beam:beam-vendor-calcite-1_26_0:0.1",
         woodstox_core_asl                           : "org.codehaus.woodstox:woodstox-core-asl:4.4.1",
         zstd_jni                                    : "com.github.luben:zstd-jni:1.4.5-2",
         quickcheck_core                             : "com.pholser:junit-quickcheck-core:$quickcheck_version",
@@ -1043,6 +1043,7 @@ class BeamModulePlugin implements Plugin<Project> {
         }
         project.spotbugs {
           excludeFilter = project.rootProject.file('sdks/java/build-tools/src/main/resources/beam/spotbugs-filter.xml')
+          jvmArgs = ['-Xmx12g']
         }
         project.tasks.withType(com.github.spotbugs.snom.SpotBugsTask) {
           reports {

sdks/java/extensions/sql/build.gradle

@@ -56,7 +56,7 @@ dependencies {
   javacc "net.java.dev.javacc:javacc:4.0"
   fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
   fmppTask "org.freemarker:freemarker:2.3.31"
-  fmppTemplates library.java.vendored_calcite_1_20_0
+  fmppTemplates library.java.vendored_calcite_1_26_0
   compile project(path: ":sdks:java:core", configuration: "shadow")
   compile project(":sdks:java:extensions:join-library")
   permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761
@@ -66,7 +66,7 @@ dependencies {
   compile library.java.commons_csv
   compile library.java.jackson_databind
   compile library.java.joda_time
-  compile library.java.vendored_calcite_1_20_0
+  compile library.java.vendored_calcite_1_26_0
   compile "com.alibaba:fastjson:1.2.69"
   compile "org.codehaus.janino:janino:3.0.11"
   compile "org.codehaus.janino:commons-compiler:3.0.11"
@@ -83,7 +83,7 @@ dependencies {
   provided library.java.jackson_dataformat_xml
   provided library.java.hadoop_client
   provided library.java.kafka_clients
-  testCompile library.java.vendored_calcite_1_20_0
+  testCompile library.java.vendored_calcite_1_26_0
   testCompile library.java.vendored_guava_26_0_jre
   testCompile library.java.junit
   testCompile library.java.quickcheck_core
@@ -123,11 +123,11 @@ task copyFmppTemplatesFromCalciteCore(type: Copy) {
   into "${project.buildDir}/templates-fmpp"
   filter{
     line ->
-      line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.')
+      line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.')
   }
   filter{
     line ->
-      line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.')
+      line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.')
   }
 }
 

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java

@@ -31,7 +31,7 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.base.Strings;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.base.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java

@@ -46,7 +46,7 @@
 import org.apache.beam.sdk.extensions.sql.meta.provider.pubsub.PubsubTableProvider;
 import org.apache.beam.sdk.extensions.sql.meta.provider.text.TextTableProvider;
 import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
 import org.checkerframework.checker.nullness.qual.Nullable;

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java

@@ -28,8 +28,8 @@
 import org.apache.beam.sdk.schemas.Schema.Field;
 import org.apache.beam.sdk.schemas.Schema.FieldType;
 import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.base.Strings;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.base.Strings;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.collect.ImmutableMap;
 
 @Experimental(Kind.SCHEMAS)
 @SuppressWarnings({

sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java

@@ -21,12 +21,13 @@
 import static org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLineTestingUtils.toLines;
 import static org.hamcrest.CoreMatchers.everyItem;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.oneOf;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.util.Arrays;
 import java.util.List;
-import org.hamcrest.collection.IsIn;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -94,7 +95,7 @@ public void testSqlLine_select() throws Exception {
     List<List<String>> lines = toLines(byteArrayOutputStream);
     assertThat(
         Arrays.asList(Arrays.asList("3", "hello", "2018-05-28")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -104,7 +105,7 @@ public void testSqlLine_selectFromTable() throws Exception {
         buildArgs(
             "CREATE EXTERNAL TABLE table_test (col_a VARCHAR, col_b VARCHAR, "
                 + "col_c VARCHAR, col_x TINYINT, col_y INT, col_z BIGINT) TYPE 'test';",
-            "INSERT INTO table_test VALUES ('a', 'b', 'c', 1, 2, 3);",
+            "INSERT INTO table_test VALUES ('a', 'b', 'c', 1, 2, CAST(3 AS BIGINT));",
             "SELECT * FROM table_test;");
 
     BeamSqlLine.runSqlLine(args, null, byteArrayOutputStream, null);
@@ -114,7 +115,7 @@ public void testSqlLine_selectFromTable() throws Exception {
         Arrays.asList(
             Arrays.asList("col_a", "col_b", "col_c", "col_x", "col_y", "col_z"),
             Arrays.asList("a", "b", "c", "1", "2", "3")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -129,8 +130,7 @@ public void testSqlLine_insertSelect() throws Exception {
     BeamSqlLine.runSqlLine(args, null, byteArrayOutputStream, null);
 
     List<List<String>> lines = toLines(byteArrayOutputStream);
-    assertThat(
-        Arrays.asList(Arrays.asList("3", "hello")), everyItem(IsIn.isOneOf(lines.toArray())));
+    assertThat(Arrays.asList(Arrays.asList("3", "hello")), everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -149,7 +149,7 @@ public void testSqlLine_GroupBy() throws Exception {
     List<List<String>> lines = toLines(byteArrayOutputStream);
     assertThat(
         Arrays.asList(Arrays.asList("3", "2"), Arrays.asList("4", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -169,7 +169,7 @@ public void testSqlLine_fixedWindow() throws Exception {
     assertThat(
         Arrays.asList(
             Arrays.asList("2018-07-01 21:26:06", "1"), Arrays.asList("2018-07-01 21:26:07", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -195,6 +195,6 @@ public void testSqlLine_slidingWindow() throws Exception {
             Arrays.asList("2018-07-01 21:26:09", "2"),
             Arrays.asList("2018-07-01 21:26:10", "2"),
             Arrays.asList("2018-07-01 21:26:11", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 }

sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java

@@ -46,9 +46,9 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.plan.RelOptRule;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.tools.RuleSet;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.tools.RuleSets;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.plan.RelOptRule;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.tools.RuleSet;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.tools.RuleSets;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 import org.junit.Before;
 import org.junit.BeforeClass;