[WIP][BEAM-9379] Update calcite to 1.26

.gitignore

@@ -16,6 +16,10 @@ sdks/**/vendor/**/*
 runners/**/vendor/**/*
 **/.gradletasknamecache
 
+# Ignore sources generated into the main tree
+**/src/main/generated/**
+**/src/test/generated_tests/**
+
 # Ignore files generated by the Maven build process.
 **/bin/**/*
 **/dependency-reduced-pom.xml
@@ -96,3 +100,8 @@ sdks/python/coverage.xml
 
 .pytest_cache
 .pytest_cache/**/*
+
+
+# Ignore files the developer has laying around
+**/*.patch
+**/*.diff

CHANGES.md

@@ -139,6 +139,7 @@
   This has not enabled by default to preserve backwards compatibility; use the
   `--type_check_additional=ptransform_fn` flag to enable. It may be enabled by
   default in future versions of Beam.
+* Updated Calcite to version 1.26
 
 ## Breaking Changes
 

build.gradle

@@ -54,6 +54,10 @@ rat {
 
     // Default eclipse excludes neglect subprojects
 
+    // Generated AutoValue java sources outside the build directory
+    "**/src/main/generated/**/AutoValue_*.java",
+    "**/src/test/generated_tests/**/AutoValue_*.java",
+
     // Proto/grpc generated wrappers
     "**/apache_beam/portability/api/*_pb2*.py",
     "**/go/pkg/beam/**/*.pb.go",
@@ -74,6 +78,7 @@ rat {
     "**/Gemfile.lock",
     "**/Rakefile",
     "**/.htaccess",
+    "website/www/node_modules/**/*",
     "website/www/site/assets/scss/_bootstrap.scss",
     "website/www/site/assets/scss/bootstrap/**/*",
     "website/www/site/static/images/mascot/*.ai",

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -616,7 +616,7 @@ class BeamModulePlugin implements Plugin<Project> {
         vendored_bytebuddy_1_10_8                   : "org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1",
         vendored_grpc_1_26_0                        : "org.apache.beam:beam-vendor-grpc-1_26_0:0.3",
         vendored_guava_26_0_jre                     : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1",
-        vendored_calcite_1_20_0                     : "org.apache.beam:beam-vendor-calcite-1_20_0:0.1",
+        vendored_calcite_1_26_0                     : "org.apache.beam:beam-vendor-calcite-1_26_0:0.1-SNAPSHOT", // FIXME: Should be 0.1 ...
         woodstox_core_asl                           : "org.codehaus.woodstox:woodstox-core-asl:4.4.1",
         zstd_jni                                    : "com.github.luben:zstd-jni:1.4.5-2",
         quickcheck_core                             : "com.pholser:junit-quickcheck-core:$quickcheck_version",

runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/OrderedCodeTest.java

@@ -148,7 +148,7 @@ public void testSignedNumIncreasing_write() {
    * Convert a string of hex digits (e.g. "3fbf") to a byte[] (e.g. { (byte) 0x3f, (byte) 0xbf }).
    */
   private static byte[] bytesFromHexString(String hexDigits) {
-    return BaseEncoding.base16().lowerCase().decode(hexDigits);
+    return new ByteString(BaseEncoding.base16().lowerCase().decode(hexDigits));
   }
 
   /**

sdks/java/extensions/sql/build.gradle

@@ -59,15 +59,15 @@ dependencies {
   javacc "net.java.dev.javacc:javacc:4.0"
   fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
   fmppTask "org.freemarker:freemarker:2.3.28"
-  fmppTemplates library.java.vendored_calcite_1_20_0
+  fmppTemplates library.java.vendored_calcite_1_26_0
   compile project(":sdks:java:core")
   compile project(":sdks:java:extensions:join-library")
   compile project(path: ":runners:direct-java", configuration: "shadow")
   compile library.java.commons_codec
   compile library.java.commons_csv
   compile library.java.jackson_databind
   compile library.java.joda_time
-  compile library.java.vendored_calcite_1_20_0
+  compile library.java.vendored_calcite_1_26_0
   compile "com.alibaba:fastjson:1.2.69"
   compile "org.codehaus.janino:janino:3.0.11"
   compile "org.codehaus.janino:commons-compiler:3.0.11"
@@ -79,7 +79,7 @@ dependencies {
   provided library.java.jackson_dataformat_xml
   provided library.java.hadoop_client
   provided library.java.kafka_clients
-  testCompile library.java.vendored_calcite_1_20_0
+  testCompile library.java.vendored_calcite_1_26_0
   testCompile library.java.vendored_guava_26_0_jre
   testCompile library.java.junit
   testCompile library.java.hamcrest_core
@@ -119,11 +119,11 @@ task copyFmppTemplatesFromCalciteCore(type: Copy) {
   into "${project.buildDir}/templates-fmpp"
   filter{
     line ->
-      line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.')
+      line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.')
   }
   filter{
     line ->
-      line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.')
+      line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.')
   }
 }
 

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/example/BeamSqlDataCatalogExample.java

@@ -31,7 +31,7 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.base.Strings;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.base.Strings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/DataCatalogTableProvider.java

@@ -42,7 +42,7 @@
 import org.apache.beam.sdk.extensions.sql.meta.provider.pubsub.PubsubTableProvider;
 import org.apache.beam.sdk.extensions.sql.meta.provider.text.TextTableProvider;
 import org.apache.beam.sdk.schemas.Schema;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects;
 import org.checkerframework.checker.nullness.qual.Nullable;
 

sdks/java/extensions/sql/datacatalog/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datacatalog/SchemaUtils.java

@@ -28,8 +28,8 @@
 import org.apache.beam.sdk.schemas.Schema.Field;
 import org.apache.beam.sdk.schemas.Schema.FieldType;
 import org.apache.beam.sdk.schemas.logicaltypes.SqlTypes;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.base.Strings;
-import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.base.Strings;
+import org.apache.beam.vendor.calcite.v1_26_0.com.google.common.collect.ImmutableMap;
 
 @Experimental(Kind.SCHEMAS)
 class SchemaUtils {

sdks/java/extensions/sql/jdbc/src/test/java/org/apache/beam/sdk/extensions/sql/jdbc/BeamSqlLineTest.java

@@ -20,13 +20,14 @@
 import static org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLineTestingUtils.buildArgs;
 import static org.apache.beam.sdk.extensions.sql.jdbc.BeamSqlLineTestingUtils.toLines;
 import static org.hamcrest.CoreMatchers.everyItem;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.oneOf;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.util.Arrays;
 import java.util.List;
-import org.hamcrest.collection.IsIn;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -94,7 +95,7 @@ public void testSqlLine_select() throws Exception {
     List<List<String>> lines = toLines(byteArrayOutputStream);
     assertThat(
         Arrays.asList(Arrays.asList("3", "hello", "2018-05-28")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -114,7 +115,7 @@ public void testSqlLine_selectFromTable() throws Exception {
         Arrays.asList(
             Arrays.asList("col_a", "col_b", "col_c", "col_x", "col_y", "col_z"),
             Arrays.asList("a", "b", "c", "1", "2", "3")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines))));
   }
 
   @Test
@@ -129,8 +130,7 @@ public void testSqlLine_insertSelect() throws Exception {
     BeamSqlLine.runSqlLine(args, null, byteArrayOutputStream, null);
 
     List<List<String>> lines = toLines(byteArrayOutputStream);
-    assertThat(
-        Arrays.asList(Arrays.asList("3", "hello")), everyItem(IsIn.isOneOf(lines.toArray())));
+    assertThat(Arrays.asList(Arrays.asList("3", "hello")), everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -149,7 +149,7 @@ public void testSqlLine_GroupBy() throws Exception {
     List<List<String>> lines = toLines(byteArrayOutputStream);
     assertThat(
         Arrays.asList(Arrays.asList("3", "2"), Arrays.asList("4", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -169,7 +169,7 @@ public void testSqlLine_fixedWindow() throws Exception {
     assertThat(
         Arrays.asList(
             Arrays.asList("2018-07-01 21:26:06", "1"), Arrays.asList("2018-07-01 21:26:07", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 
   @Test
@@ -195,6 +195,6 @@ public void testSqlLine_slidingWindow() throws Exception {
             Arrays.asList("2018-07-01 21:26:09", "2"),
             Arrays.asList("2018-07-01 21:26:10", "2"),
             Arrays.asList("2018-07-01 21:26:11", "1")),
-        everyItem(IsIn.isOneOf(lines.toArray())));
+        everyItem(is(oneOf(lines.toArray()))));
   }
 }

sdks/java/extensions/sql/perf-tests/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigquery/BigQueryIOPushDownIT.java

@@ -46,9 +46,9 @@
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.Row;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.plan.RelOptRule;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.tools.RuleSet;
-import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.tools.RuleSets;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.plan.RelOptRule;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.tools.RuleSet;
+import org.apache.beam.vendor.calcite.v1_26_0.org.apache.calcite.tools.RuleSets;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 import org.junit.Before;
 import org.junit.BeforeClass;