From e51cf6eff42885f0b93d9c7cc1bf75d207eceba1 Mon Sep 17 00:00:00 2001 From: Mike Kaplinskiy Date: Sun, 15 Apr 2018 14:34:55 -0700 Subject: [PATCH 1/2] Add GoogleSheetsOptions to allow bigquery to use tables from Google Sheets. --- .../bigquery/ExternalTableDefinition.java | 9 ++ .../google/cloud/bigquery/FormatOptions.java | 10 ++ .../cloud/bigquery/GoogleSheetsOptions.java | 123 ++++++++++++++++++ .../cloud/bigquery/FormatOptionsTest.java | 2 + .../bigquery/GoogleSheetsOptionsTest.java | 65 +++++++++ 5 files changed, 209 insertions(+) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/GoogleSheetsOptions.java create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 497a8be2cfbb..d63918b9bb45 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -236,6 +236,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) { externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb()); } + if (getFormatOptions() != null && FormatOptions.GOOGLE_SHEETS.equals(getFormatOptions().getType())) { + externalConfigurationPb.setGoogleSheetsOptions(((GoogleSheetsOptions) getFormatOptions()).toPb()); + } if (getAutodetect() != null) { externalConfigurationPb.setAutodetect(getAutodetect()); } @@ -340,6 +343,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } + if (externalDataConfiguration.getGoogleSheetsOptions() != null) { + builder.setFormatOptions(GoogleSheetsOptions.fromPb(externalDataConfiguration.getGoogleSheetsOptions())); + } builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords()); builder.setAutodetect(externalDataConfiguration.getAutodetect()); } @@ -367,6 +373,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getCsvOptions() != null) { builder.setFormatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } + if (externalDataConfiguration.getGoogleSheetsOptions() != null) { + builder.setFormatOptions(GoogleSheetsOptions.fromPb(externalDataConfiguration.getGoogleSheetsOptions())); + } if (externalDataConfiguration.getMaxBadRecords() != null) { builder.setMaxBadRecords(externalDataConfiguration.getMaxBadRecords()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java index 841a8318e31f..439039f0f071 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java @@ -33,6 +33,7 @@ public class FormatOptions implements Serializable { static final String JSON = "NEWLINE_DELIMITED_JSON"; static final String DATASTORE_BACKUP = "DATASTORE_BACKUP"; static final String AVRO = "AVRO"; + static final String GOOGLE_SHEETS = "GOOGLE_SHEETS"; private static final long serialVersionUID = -443376052020423691L; private final String type; @@ -95,6 +96,13 @@ public static FormatOptions avro() { return new FormatOptions(AVRO); } + /** + * Default options for GOOGLE_SHEETS format. + */ + public static FormatOptions googleSheets() { + return GoogleSheetsOptions.newBuilder().build(); + } + /** * Default options for the provided format. */ @@ -103,6 +111,8 @@ public static FormatOptions of(String format) { return csv(); } else if (format.equals(DATASTORE_BACKUP)) { return datastoreBackup(); + } else if (format.equals(GOOGLE_SHEETS)) { + return googleSheets(); } return new FormatOptions(format); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/GoogleSheetsOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/GoogleSheetsOptions.java new file mode 100644 index 000000000000..b882f81bfbaf --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/GoogleSheetsOptions.java @@ -0,0 +1,123 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; + +/** + * Google BigQuery options for the Google Sheets format. + */ +public final class GoogleSheetsOptions extends FormatOptions { + + private static final long serialVersionUID = 1837436979033106123L; + + private final Long skipLeadingRows; + + public static final class Builder { + + private Long skipLeadingRows; + + private Builder() {} + + private Builder(GoogleSheetsOptions options) { + this.skipLeadingRows = options.skipLeadingRows; + } + + /** + * Sets the number of rows at the top of a sheet that BigQuery will skip when reading the + * data. The default value is 0. This property is useful if you have header rows + * that should be skipped. + */ + public Builder setSkipLeadingRows(long skipLeadingRows) { + this.skipLeadingRows = skipLeadingRows; + return this; + } + + /** + * Creates a {@link GoogleSheetsOptions} object. + */ + public GoogleSheetsOptions build() { + return new GoogleSheetsOptions(this); + } + } + + private GoogleSheetsOptions(Builder builder) { + super(FormatOptions.GOOGLE_SHEETS); + this.skipLeadingRows = builder.skipLeadingRows; + } + + + /** + * Returns the number of rows at the top of a sheet that BigQuery will skip when reading the + * data. + */ + public Long getSkipLeadingRows() { + return skipLeadingRows; + } + + /** + * Returns a builder for the {@link GoogleSheetsOptions} object. + */ + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", getType()) + .add("skipLeadingRows", skipLeadingRows) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(getType(), skipLeadingRows); + } + + @Override + public boolean equals(Object obj) { + return obj == this + || obj instanceof GoogleSheetsOptions + && Objects.equals(toPb(), ((GoogleSheetsOptions) obj).toPb()); + } + + com.google.api.services.bigquery.model.GoogleSheetsOptions toPb() { + com.google.api.services.bigquery.model.GoogleSheetsOptions options = + new com.google.api.services.bigquery.model.GoogleSheetsOptions(); + options.setSkipLeadingRows(skipLeadingRows); + return options; + } + + + /** + * Returns a builder for a {@link GoogleSheetsOptions} object. + */ + public static Builder newBuilder() { + return new Builder(); + } + + static GoogleSheetsOptions fromPb(com.google.api.services.bigquery.model.GoogleSheetsOptions options) { + Builder builder = newBuilder(); + if (options.getSkipLeadingRows() != null) { + builder.setSkipLeadingRows(options.getSkipLeadingRows()); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FormatOptionsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FormatOptionsTest.java index 276ea6221307..651c25d0dc5c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FormatOptionsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/FormatOptionsTest.java @@ -41,6 +41,7 @@ public void testFactoryMethods() { assertEquals(FormatOptions.JSON, FormatOptions.json().getType()); assertEquals(FormatOptions.DATASTORE_BACKUP, FormatOptions.datastoreBackup().getType()); assertEquals(FormatOptions.AVRO, FormatOptions.avro().getType()); + assertEquals(FormatOptions.GOOGLE_SHEETS, FormatOptions.googleSheets().getType()); } @Test @@ -52,5 +53,6 @@ public void testEquals() { assertEquals(FormatOptions.datastoreBackup(), FormatOptions.datastoreBackup()); assertEquals(FormatOptions.datastoreBackup().hashCode(), FormatOptions.datastoreBackup().hashCode()); + assertEquals(FormatOptions.googleSheets(), FormatOptions.googleSheets()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java new file mode 100644 index 000000000000..1cc213e41f86 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class GoogleSheetsOptionsTest { + + private static final long SKIP_LEADING_ROWS = 42L; + private static final GoogleSheetsOptions GOOGLE_SHEETS_OPTIONS = GoogleSheetsOptions.newBuilder() + .setSkipLeadingRows(SKIP_LEADING_ROWS) + .build(); + + @Test + public void testToBuilder() { + compareGoogleSheetsOptions(GOOGLE_SHEETS_OPTIONS, GOOGLE_SHEETS_OPTIONS.toBuilder().build()); + GoogleSheetsOptions googleSheetsOptions = GOOGLE_SHEETS_OPTIONS.toBuilder() + .setSkipLeadingRows(123) + .build(); + assertEquals(123, (long) googleSheetsOptions.getSkipLeadingRows()); + googleSheetsOptions = googleSheetsOptions.toBuilder().setSkipLeadingRows(SKIP_LEADING_ROWS).build(); + compareGoogleSheetsOptions(GOOGLE_SHEETS_OPTIONS, googleSheetsOptions); + } + + @Test + public void testToBuilderIncomplete() { + GoogleSheetsOptions googleSheetsOptions = GoogleSheetsOptions.newBuilder().build(); + assertEquals(googleSheetsOptions, googleSheetsOptions.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(FormatOptions.GOOGLE_SHEETS, GOOGLE_SHEETS_OPTIONS.getType()); + assertEquals(SKIP_LEADING_ROWS, (long) GOOGLE_SHEETS_OPTIONS.getSkipLeadingRows()); + } + + + @Test + public void testToAndFromPb() { + compareGoogleSheetsOptions(GOOGLE_SHEETS_OPTIONS, GoogleSheetsOptions.fromPb(GOOGLE_SHEETS_OPTIONS.toPb())); + GoogleSheetsOptions googleSheetsOptions = GoogleSheetsOptions.newBuilder().build(); + compareGoogleSheetsOptions(googleSheetsOptions, GoogleSheetsOptions.fromPb(googleSheetsOptions.toPb())); + } + + private void compareGoogleSheetsOptions(GoogleSheetsOptions expected, GoogleSheetsOptions value) { + assertEquals(expected, value); + assertEquals(expected.getSkipLeadingRows(), value.getSkipLeadingRows()); + } +} From c9bac4b8cd82cf5269b48e8e5ea358a364af5526 Mon Sep 17 00:00:00 2001 From: Mike Kaplinskiy Date: Mon, 16 Apr 2018 11:17:41 -0700 Subject: [PATCH 2/2] Switch to using assertThat --- .../cloud/bigquery/GoogleSheetsOptionsTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java index 1cc213e41f86..bf9488ad91db 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/GoogleSheetsOptionsTest.java @@ -18,7 +18,7 @@ import org.junit.Test; -import static org.junit.Assert.assertEquals; +import static com.google.common.truth.Truth.assertThat; public class GoogleSheetsOptionsTest { @@ -33,7 +33,7 @@ public void testToBuilder() { GoogleSheetsOptions googleSheetsOptions = GOOGLE_SHEETS_OPTIONS.toBuilder() .setSkipLeadingRows(123) .build(); - assertEquals(123, (long) googleSheetsOptions.getSkipLeadingRows()); + assertThat(googleSheetsOptions.getSkipLeadingRows()).isEqualTo(123); googleSheetsOptions = googleSheetsOptions.toBuilder().setSkipLeadingRows(SKIP_LEADING_ROWS).build(); compareGoogleSheetsOptions(GOOGLE_SHEETS_OPTIONS, googleSheetsOptions); } @@ -41,13 +41,13 @@ public void testToBuilder() { @Test public void testToBuilderIncomplete() { GoogleSheetsOptions googleSheetsOptions = GoogleSheetsOptions.newBuilder().build(); - assertEquals(googleSheetsOptions, googleSheetsOptions.toBuilder().build()); + assertThat(googleSheetsOptions.toBuilder().build()).isEqualTo(googleSheetsOptions); } @Test public void testBuilder() { - assertEquals(FormatOptions.GOOGLE_SHEETS, GOOGLE_SHEETS_OPTIONS.getType()); - assertEquals(SKIP_LEADING_ROWS, (long) GOOGLE_SHEETS_OPTIONS.getSkipLeadingRows()); + assertThat(GOOGLE_SHEETS_OPTIONS.getType()).isEqualTo(FormatOptions.GOOGLE_SHEETS); + assertThat(GOOGLE_SHEETS_OPTIONS.getSkipLeadingRows()).isEqualTo(SKIP_LEADING_ROWS); } @@ -59,7 +59,7 @@ public void testToAndFromPb() { } private void compareGoogleSheetsOptions(GoogleSheetsOptions expected, GoogleSheetsOptions value) { - assertEquals(expected, value); - assertEquals(expected.getSkipLeadingRows(), value.getSkipLeadingRows()); + assertThat(value).isEqualTo(expected); + assertThat(value.getSkipLeadingRows()).isEqualTo(expected.getSkipLeadingRows()); } }