From 15edc40db02d198f29d3873bc8b0590259c0fc0d Mon Sep 17 00:00:00 2001 From: Andrew Pilloud Date: Fri, 3 Sep 2021 16:12:19 -0700 Subject: [PATCH] [BEAM-12852] Revert BigTable changes, just cast to bigint This is a partal revert of commit cfcde44. --- .../bigtable/BigtableTableFlatTest.java | 6 ++--- .../provider/bigtable/BigtableTableIT.java | 2 +- .../bigtable/BigtableTableTestUtils.java | 24 +++++++++---------- .../bigtable/BigtableTableWithRowsTest.java | 6 ++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java index f17a96e281cd..817570ab2cd7 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableFlatTest.java @@ -106,7 +106,7 @@ public void testSimpleSelectFlat() { BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new BigtableTableProvider()); sqlEnv.executeDdl(createFlatTableString(tableId, location(tableId))); - String query = "SELECT key, boolColumn, intColumn, stringColumn, doubleColumn FROM flatTable"; + String query = "SELECT key, boolColumn, longColumn, stringColumn, doubleColumn FROM flatTable"; sqlEnv.parseQuery(query); PCollection queryOutput = @@ -145,8 +145,8 @@ public void testSimpleInsert() { sqlEnv.executeDdl(createFlatTableString(tableId, location(tableId))); String query = - "INSERT INTO beamWriteTable(key, boolColumn, intColumn, stringColumn, doubleColumn) " - + "VALUES ('key', TRUE, 10, 'stringValue', 5.5)"; + "INSERT INTO beamWriteTable(key, boolColumn, longColumn, stringColumn, doubleColumn) " + + "VALUES ('key', TRUE, CAST(10 AS bigint), 'stringValue', 5.5)"; BeamSqlRelUtils.toPCollection(writePipeline, sqlEnv.parseQuery(query)); writePipeline.run().waitUntilFinish(); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java index 4d74a0657ae7..0a9ab2ffc94b 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java @@ -100,7 +100,7 @@ private void writeData() { String query = String.format( "INSERT INTO `%s`(key, boolColumn, longColumn, stringColumn, doubleColumn) " - + "VALUES ('key1', FALSE, 1, 'string1', 1.0)", + + "VALUES ('key1', FALSE, CAST(1 as bigint), 'string1', 1.0)", TABLE_ID); BeamSqlRelUtils.toPCollection(p, sqlEnv.parseQuery(query)); diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java index 5cc358aa6258..f318676f6ba1 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java @@ -39,7 +39,7 @@ import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.values.Row; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Ints; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs; import org.checkerframework.checker.nullness.qual.Nullable; class BigtableTableTestUtils { @@ -48,12 +48,12 @@ class BigtableTableTestUtils { static final String KEY2 = "key2"; static final String BOOL_COLUMN = "boolColumn"; - static final String INT_COLUMN = "intColumn"; + static final String LONG_COLUMN = "longColumn"; static final String STRING_COLUMN = "stringColumn"; static final String DOUBLE_COLUMN = "doubleColumn"; static final String FAMILY_TEST = "familyTest"; - static final Schema INT_COLUMN_SCHEMA = + static final Schema LONG_COLUMN_SCHEMA = Schema.builder() .addInt64Field(VALUE) .addInt64Field(TIMESTAMP_MICROS) @@ -63,7 +63,7 @@ class BigtableTableTestUtils { static final Schema TEST_FAMILY_SCHEMA = Schema.builder() .addBooleanField(BOOL_COLUMN) - .addRowField(INT_COLUMN, INT_COLUMN_SCHEMA) + .addRowField(LONG_COLUMN, LONG_COLUMN_SCHEMA) .addArrayField(STRING_COLUMN, Schema.FieldType.STRING) .addDoubleField(DOUBLE_COLUMN) .build(); @@ -75,7 +75,7 @@ class BigtableTableTestUtils { Schema.builder() .addStringField(KEY) .addBooleanField(BOOL_COLUMN) - .addInt64Field(INT_COLUMN) + .addInt64Field(LONG_COLUMN) .addStringField(STRING_COLUMN) .addDoubleField(DOUBLE_COLUMN) .build(); @@ -88,7 +88,7 @@ static String createFlatTableString(String table, String location) { "CREATE EXTERNAL TABLE `%s`( \n" + " key VARCHAR NOT NULL, \n" + " boolColumn BOOLEAN NOT NULL, \n" - + " intColumn BIGINT NOT NULL, \n" + + " longColumn BIGINT NOT NULL, \n" + " stringColumn VARCHAR NOT NULL, \n" + " doubleColumn DOUBLE NOT NULL \n" + ") \n" @@ -105,7 +105,7 @@ static String createFullTableString(String tableId, String location) { + " key VARCHAR NOT NULL, \n" + " familyTest ROW< \n" + " boolColumn BOOLEAN NOT NULL, \n" - + " intColumn ROW< \n" + + " longColumn ROW< \n" + " val BIGINT NOT NULL, \n" + " timestampMicros BIGINT NOT NULL, \n" + " labels ARRAY NOT NULL \n" @@ -123,7 +123,7 @@ static Schema expectedFullSchema() { return Schema.builder() .addStringField(KEY) .addBooleanField(BOOL_COLUMN) - .addInt64Field("intValue") + .addInt64Field("longValue") .addInt64Field(TIMESTAMP_MICROS) .addArrayField(LABELS, Schema.FieldType.STRING) .addArrayField(STRING_COLUMN, Schema.FieldType.STRING) @@ -155,7 +155,7 @@ static String location( } static String columnsMappingString() { - return "familyTest:boolColumn,familyTest:intColumn,familyTest:doubleColumn," + return "familyTest:boolColumn,familyTest:longColumn,familyTest:doubleColumn," + "familyTest:stringColumn"; } @@ -170,7 +170,7 @@ static com.google.bigtable.v2.Row bigTableRow() { ImmutableList.of( column("boolColumn", booleanToByteArray(true)), column("doubleColumn", doubleToByteArray(5.5)), - column("intColumn", Ints.toByteArray(10)), + column("longColumn", Longs.toByteArray(10L)), column("stringColumn", "stringValue".getBytes(UTF_8))); Family family = Family.newBuilder().setName("familyTest").addAllColumns(columns).build(); return com.google.bigtable.v2.Row.newBuilder() @@ -226,8 +226,8 @@ private static void writeRow(String key, String table, BigtableClientWrapper cli clientWrapper.writeRow(key, table, FAMILY_TEST, STRING_COLUMN, "string1".getBytes(UTF_8), NOW); clientWrapper.writeRow( key, table, FAMILY_TEST, STRING_COLUMN, "string2".getBytes(UTF_8), LATER); - clientWrapper.writeRow(key, table, FAMILY_TEST, INT_COLUMN, longToByteArray(1L), NOW); - clientWrapper.writeRow(key, table, FAMILY_TEST, INT_COLUMN, longToByteArray(2L), LATER); + clientWrapper.writeRow(key, table, FAMILY_TEST, LONG_COLUMN, longToByteArray(1L), NOW); + clientWrapper.writeRow(key, table, FAMILY_TEST, LONG_COLUMN, longToByteArray(2L), LATER); clientWrapper.writeRow(key, table, FAMILY_TEST, DOUBLE_COLUMN, doubleToByteArray(1.10), NOW); clientWrapper.writeRow(key, table, FAMILY_TEST, DOUBLE_COLUMN, doubleToByteArray(2.20), LATER); } diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java index a170ebfa08f3..4b60eb13c894 100644 --- a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java +++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableWithRowsTest.java @@ -96,9 +96,9 @@ public void testSimpleSelect() { String query = "SELECT key, \n" + " bt.familyTest.boolColumn, \n" - + " bt.familyTest.intColumn.val AS intValue, \n" - + " bt.familyTest.intColumn.timestampMicros, \n" - + " bt.familyTest.intColumn.labels, \n" + + " bt.familyTest.longColumn.val AS longValue, \n" + + " bt.familyTest.longColumn.timestampMicros, \n" + + " bt.familyTest.longColumn.labels, \n" + " bt.familyTest.stringColumn, \n" + " bt.familyTest.doubleColumn \n" + "FROM beamTable bt";