From fdf7e1b64eb7c8d632f9fe5c3e51aa9cd9ca5b52 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 1 Mar 2019 19:55:46 +0530 Subject: [PATCH 1/4] Fix useAvroLogicalTypes in LoadJobConfiguration --- .../cloud/bigquery/LoadConfiguration.java | 10 +++++++ .../cloud/bigquery/LoadJobConfiguration.java | 20 ++++++++++++- .../bigquery/LoadJobConfigurationTest.java | 29 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java index 3f3382420569..de494567df92 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java @@ -118,6 +118,13 @@ interface Builder { /** Sets the clustering specification for the destination table. */ Builder setClustering(Clustering clustering); + /** + * [Optional] If FormatOptions is set to "AVRO", indicates whether to enable interpreting logical + * types into their corresponding types (ie. TIMESTAMP), instead of only using their raw types + * (ie. INTEGER). The value may be {@code null}. + */ + Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes); + LoadConfiguration build(); } @@ -204,6 +211,9 @@ interface Builder { /** Returns the clustering specification for the definition table. */ Clustering getClustering(); + /** Returns the True/False indicates whether to convert logical types. */ + Boolean getUseAvroLogicalTypes(); + /** Returns a builder for the load configuration object. */ Builder toBuilder(); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index d11f17081e2e..0f7e81a2bb13 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -49,6 +49,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final Boolean autodetect; private final TimePartitioning timePartitioning; private final Clustering clustering; + private final Boolean useAvroLogicalTypes; public static final class Builder extends JobConfiguration.Builder implements LoadConfiguration.Builder { @@ -68,6 +69,7 @@ public static final class Builder extends JobConfiguration.Builder getSchemaUpdateOptions() { return schemaUpdateOptions; @@ -383,7 +399,8 @@ ToStringHelper toStringHelper() { .add("schemaUpdateOptions", schemaUpdateOptions) .add("autodetect", autodetect) .add("timePartitioning", timePartitioning) - .add("clustering", clustering); + .add("clustering", clustering) + .add("useAvroLogicalTypes", useAvroLogicalTypes); } @Override @@ -464,6 +481,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (clustering != null) { loadConfigurationPb.setClustering(clustering.toPb()); } + loadConfigurationPb.setUseAvroLogicalTypes(useAvroLogicalTypes); return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 85184d88e069..e78824b644e6 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -41,6 +41,7 @@ public class LoadJobConfigurationTest { private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; private static final Integer MAX_BAD_RECORDS = 42; private static final String FORMAT = "CSV"; + private static final String AVRO_FORMATE = "AVRO"; private static final Boolean IGNORE_UNKNOWN_VALUES = true; private static final Field FIELD_SCHEMA = Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER) @@ -52,6 +53,7 @@ public class LoadJobConfigurationTest { ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); private static final Boolean AUTODETECT = true; + private static final Boolean USERAVROLOGICALTYPES = true; private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); @@ -86,6 +88,21 @@ public class LoadJobConfigurationTest { .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) .setAutodetect(AUTODETECT) .build(); + private static final LoadJobConfiguration LOAD_CONFIGURATION_AVRO = + LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) + .setCreateDisposition(CREATE_DISPOSITION) + .setWriteDisposition(WRITE_DISPOSITION) + .setFormatOptions(FormatOptions.avro()) + .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .setMaxBadRecords(MAX_BAD_RECORDS) + .setSchema(TABLE_SCHEMA) + .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) + .setAutodetect(AUTODETECT) + .setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) + .setUseAvroLogicalTypes(USERAVROLOGICALTYPES) + .build(); @Test public void testToBuilder() { @@ -109,6 +126,17 @@ public void testToBuilder() { assertEquals("newTable", configurationBackup.getDestinationTable().getTable()); configurationBackup = configurationBackup.toBuilder().setDestinationTable(TABLE_ID).build(); compareLoadJobConfiguration(LOAD_CONFIGURATION_BACKUP, configurationBackup); + + compareLoadJobConfiguration( + LOAD_CONFIGURATION_AVRO, LOAD_CONFIGURATION_AVRO.toBuilder().build()); + LoadJobConfiguration configurationAvro = + LOAD_CONFIGURATION_AVRO + .toBuilder() + .setDestinationTable(TableId.of("dataset", "newTable")) + .build(); + assertEquals("newTable", configurationAvro.getDestinationTable().getTable()); + configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build(); + compareLoadJobConfiguration(LOAD_CONFIGURATION_AVRO, configurationBackup); } @Test @@ -188,5 +216,6 @@ private void compareLoadJobConfiguration( value.getDestinationEncryptionConfiguration()); assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); assertEquals(expected.getClustering(), value.getClustering()); + assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes()); } } From 0d09a58ed93dcecad1c09e0bd163b3ac3be0503a Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 1 Mar 2019 22:23:12 +0530 Subject: [PATCH 2/4] Fix comment and added useAvroLogicalTypes in WriteChannelConfiguration --- .../cloud/bigquery/LoadConfiguration.java | 8 +++--- .../bigquery/WriteChannelConfiguration.java | 23 ++++++++++++++-- .../bigquery/LoadJobConfigurationTest.java | 3 +-- .../WriteChannelConfigurationTest.java | 27 +++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java index de494567df92..bca9565e9e36 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java @@ -119,9 +119,9 @@ interface Builder { Builder setClustering(Clustering clustering); /** - * [Optional] If FormatOptions is set to "AVRO", indicates whether to enable interpreting logical - * types into their corresponding types (ie. TIMESTAMP), instead of only using their raw types - * (ie. INTEGER). The value may be {@code null}. + * If FormatOptions is set to AVRO, you can convert logical types into their corresponding types + * (such as TIMESTAMP) instead of only using their raw types (such as INTEGER). The value may be + * {@code null}. */ Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes); @@ -211,7 +211,7 @@ interface Builder { /** Returns the clustering specification for the definition table. */ Clustering getClustering(); - /** Returns the True/False indicates whether to convert logical types. */ + /** Returns True/False. Indicates whether the logical type is converted. */ Boolean getUseAvroLogicalTypes(); /** Returns a builder for the load configuration object. */ diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java index d84d8b60a392..b99f718794ce 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java @@ -52,6 +52,7 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria private final EncryptionConfiguration destinationEncryptionConfiguration; private final TimePartitioning timePartitioning; private final Clustering clustering; + private final Boolean useAvroLogicalTypes; public static final class Builder implements LoadConfiguration.Builder { @@ -68,6 +69,7 @@ public static final class Builder implements LoadConfiguration.Builder { private EncryptionConfiguration destinationEncryptionConfiguration; private TimePartitioning timePartitioning; private Clustering clustering; + private Boolean useAvroLogicalTypes; private Builder() {} @@ -86,6 +88,7 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) { writeChannelConfiguration.destinationEncryptionConfiguration; this.timePartitioning = writeChannelConfiguration.timePartitioning; this.clustering = writeChannelConfiguration.clustering; + this.useAvroLogicalTypes = writeChannelConfiguration.useAvroLogicalTypes; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -158,6 +161,7 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur if (loadConfigurationPb.getClustering() != null) { this.clustering = Clustering.fromPb(loadConfigurationPb.getClustering()); } + this.useAvroLogicalTypes = loadConfigurationPb.getUseAvroLogicalTypes(); } @Override @@ -240,6 +244,12 @@ public Builder setClustering(Clustering clustering) { return this; } + @Override + public Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes) { + this.useAvroLogicalTypes = useAvroLogicalTypes; + return this; + } + @Override public WriteChannelConfiguration build() { return new WriteChannelConfiguration(this); @@ -260,6 +270,7 @@ protected WriteChannelConfiguration(Builder builder) { this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration; this.timePartitioning = builder.timePartitioning; this.clustering = builder.clustering; + this.useAvroLogicalTypes = builder.useAvroLogicalTypes; } @Override @@ -339,6 +350,11 @@ public Clustering getClustering() { return clustering; } + @Override + public Boolean getUseAvroLogicalTypes() { + return useAvroLogicalTypes; + } + @Override public Builder toBuilder() { return new Builder(this); @@ -358,7 +374,8 @@ MoreObjects.ToStringHelper toStringHelper() { .add("schemaUpdateOptions", schemaUpdateOptions) .add("autodetect", autodetect) .add("timePartitioning", timePartitioning) - .add("clustering", clustering); + .add("clustering", clustering) + .add("useAvroLogicalTypes", useAvroLogicalTypes); } @Override @@ -387,7 +404,8 @@ public int hashCode() { schemaUpdateOptions, autodetect, timePartitioning, - clustering); + clustering, + useAvroLogicalTypes); } WriteChannelConfiguration setProjectId(String projectId) { @@ -452,6 +470,7 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (clustering != null) { loadConfigurationPb.setClustering(clustering.toPb()); } + loadConfigurationPb.setUseAvroLogicalTypes(useAvroLogicalTypes); return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index e78824b644e6..48f41beb59c7 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -41,7 +41,6 @@ public class LoadJobConfigurationTest { private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; private static final Integer MAX_BAD_RECORDS = 42; private static final String FORMAT = "CSV"; - private static final String AVRO_FORMATE = "AVRO"; private static final Boolean IGNORE_UNKNOWN_VALUES = true; private static final Field FIELD_SCHEMA = Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER) @@ -136,7 +135,7 @@ public void testToBuilder() { .build(); assertEquals("newTable", configurationAvro.getDestinationTable().getTable()); configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build(); - compareLoadJobConfiguration(LOAD_CONFIGURATION_AVRO, configurationBackup); + compareLoadJobConfiguration(LOAD_CONFIGURATION_AVRO, configurationAvro); } @Test diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java index 9fe80612a059..39c7adea336e 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java @@ -49,6 +49,7 @@ public class WriteChannelConfigurationTest { .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); private static final Boolean AUTODETECT = true; + private static final Boolean USERAVROLOGICALTYPES = true; private static final List SCHEMA_UPDATE_OPTIONS = ImmutableList.of(JobInfo.SchemaUpdateOption.ALLOW_FIELD_ADDITION); private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); @@ -84,6 +85,21 @@ public class WriteChannelConfigurationTest { .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) .setAutodetect(AUTODETECT) .build(); + private static final WriteChannelConfiguration LOAD_CONFIGURATION_AVRO = + WriteChannelConfiguration.newBuilder(TABLE_ID) + .setCreateDisposition(CREATE_DISPOSITION) + .setWriteDisposition(WRITE_DISPOSITION) + .setNullMarker(NULL_MARKER) + .setFormatOptions(FormatOptions.avro()) + .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .setMaxBadRecords(MAX_BAD_RECORDS) + .setSchema(TABLE_SCHEMA) + .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) + .setAutodetect(AUTODETECT) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) + .setUseAvroLogicalTypes(USERAVROLOGICALTYPES) + .build(); @Test public void testToBuilder() { @@ -96,6 +112,16 @@ public void testToBuilder() { assertEquals("newTable", configuration.getDestinationTable().getTable()); configuration = configuration.toBuilder().setDestinationTable(TABLE_ID).build(); compareLoadConfiguration(LOAD_CONFIGURATION_CSV, configuration); + + compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, LOAD_CONFIGURATION_AVRO.toBuilder().build()); + WriteChannelConfiguration configurationAvro = + LOAD_CONFIGURATION_AVRO + .toBuilder() + .setDestinationTable(TableId.of("dataset", "newTable")) + .build(); + assertEquals("newTable", configurationAvro.getDestinationTable().getTable()); + configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build(); + compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, configurationAvro); } @Test @@ -191,5 +217,6 @@ private void compareLoadConfiguration( assertEquals(expected.getAutodetect(), value.getAutodetect()); assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); assertEquals(expected.getClustering(), value.getClustering()); + assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes()); } } From 32b1949df10ba0811718df09ca916d5d709beeb6 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 1 Mar 2019 22:46:54 +0530 Subject: [PATCH 3/4] Fix comment --- .../java/com/google/cloud/bigquery/LoadConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java index bca9565e9e36..a7af95203fcf 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java @@ -119,7 +119,7 @@ interface Builder { Builder setClustering(Clustering clustering); /** - * If FormatOptions is set to AVRO, you can convert logical types into their corresponding types + * If FormatOptions is set to AVRO, you can interpret logical types into their corresponding types * (such as TIMESTAMP) instead of only using their raw types (such as INTEGER). The value may be * {@code null}. */ @@ -211,7 +211,7 @@ interface Builder { /** Returns the clustering specification for the definition table. */ Clustering getClustering(); - /** Returns True/False. Indicates whether the logical type is converted. */ + /** Returns True/False. Indicates whether the logical type is interpreted. */ Boolean getUseAvroLogicalTypes(); /** Returns a builder for the load configuration object. */ From f94dae30575a526129aa956e2f93f6ba299f854f Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 1 Mar 2019 23:12:19 +0530 Subject: [PATCH 4/4] Fix code format --- .../java/com/google/cloud/bigquery/LoadConfiguration.java | 6 +++--- .../cloud/bigquery/WriteChannelConfigurationTest.java | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java index a7af95203fcf..15e82fc914c6 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java @@ -119,9 +119,9 @@ interface Builder { Builder setClustering(Clustering clustering); /** - * If FormatOptions is set to AVRO, you can interpret logical types into their corresponding types - * (such as TIMESTAMP) instead of only using their raw types (such as INTEGER). The value may be - * {@code null}. + * If FormatOptions is set to AVRO, you can interpret logical types into their corresponding + * types (such as TIMESTAMP) instead of only using their raw types (such as INTEGER). The value + * may be {@code null}. */ Builder setUseAvroLogicalTypes(Boolean useAvroLogicalTypes); diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java index 39c7adea336e..d78d3995a97f 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java @@ -115,10 +115,10 @@ public void testToBuilder() { compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, LOAD_CONFIGURATION_AVRO.toBuilder().build()); WriteChannelConfiguration configurationAvro = - LOAD_CONFIGURATION_AVRO - .toBuilder() - .setDestinationTable(TableId.of("dataset", "newTable")) - .build(); + LOAD_CONFIGURATION_AVRO + .toBuilder() + .setDestinationTable(TableId.of("dataset", "newTable")) + .build(); assertEquals("newTable", configurationAvro.getDestinationTable().getTable()); configurationAvro = configurationAvro.toBuilder().setDestinationTable(TABLE_ID).build(); compareLoadConfiguration(LOAD_CONFIGURATION_AVRO, configurationAvro);