From 4bc52c3bbc4ffb1c938376227477eca187e1e43b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 16 Nov 2016 14:03:18 -0800 Subject: [PATCH 1/2] Update BigQuery.writer snippets to get LoadStatistics from Job. Also, add an example for loading from a file, not just a string. --- .../com/google/cloud/bigquery/BigQuery.java | 39 +++++++++++-- .../bigquery/snippets/BigQuerySnippets.java | 55 ++++++++++++++++--- .../bigquery/snippets/ITBigQuerySnippets.java | 14 ++++- 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 47a30cfd32aa..b020b8f01e26 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -1014,15 +1014,42 @@ Page> listTableData(String datasetId, String tableId, * WriteChannelConfiguration.newBuilder(tableId) * .setFormatOptions(FormatOptions.csv()) * .build(); - * BaseWriteChannel writer = - * bigquery.writer(writeChannelConfiguration); + * TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration); + * // Write data to writer + * try { + * writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); + * } finally { + * writer.close(); + * } + * // Get load job + * Job job = writer.getJob(); + * job = job.waitFor(); + * LoadStatistics stats = job.getStatistics(); + * return stats.getOutputRows(); + * } + * + *

Example of writing a local file to a table. + * 

 {@code
+   * String datasetName = "my_dataset_name";
+   * String tableName = "my_table_name";
+   * ReadableByteChannel csvReader = Files.newByteChannel(FileSystems.getDefault().getPath(".", "my-data.csv"));
+   * TableId tableId = TableId.of(datasetName, tableName);
+   * WriteChannelConfiguration writeChannelConfiguration =
+   *     WriteChannelConfiguration.newBuilder(tableId)
+   *         .setFormatOptions(FormatOptions.csv())
+   *         .build();
+   * TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration);
    * // Write data to writer
    * try {
-   *   writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8)));
-   * } catch (IOException e) {
-   *   // Unable to write data
+   *   ByteStreams.copy(csvReader, writer);
+   * } finally {
+   *   writer.close();
    * }
-   * writer.close();
+   * // Get load job
+   * Job job = writer.getJob();
+   * job = job.waitFor();
+   * LoadStatistics stats = job.getStatistics();
+   * return stats.getOutputRows();
    * }
* * @throws BigQueryException upon failure diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java index 85d556861a51..3f715b03e70a 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java @@ -23,6 +23,7 @@ package com.google.cloud.examples.bigquery.snippets; import com.google.api.client.util.Charsets; +import com.google.api.services.bigquery.model.JobStatistics; import com.google.cloud.Page; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; @@ -44,6 +45,7 @@ import com.google.cloud.bigquery.JobConfiguration; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics.LoadStatistics; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryRequest; import com.google.cloud.bigquery.QueryResponse; @@ -56,14 +58,20 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.WriteChannelConfiguration; +import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.TimeoutException; /** * This class contains a number of snippets for the {@link BigQuery} interface. @@ -331,10 +339,40 @@ public Table getTableFromId(String projectId, String datasetName, String tableNa // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE "StringValue1\nStringValue2\n"] - public TableDataWriteChannel writeToTable(String datasetName, String tableName, String csvData) - throws IOException { + public long writeToTable(String datasetName, String tableName, String csvData) + throws IOException, InterruptedException, TimeoutException { // [START writeToTable] TableId tableId = TableId.of(datasetName, tableName); + WriteChannelConfiguration writeChannelConfiguration = + WriteChannelConfiguration.newBuilder(tableId) + .setFormatOptions(FormatOptions.csv()) + .build(); + TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration); + // Write data to writer + try { + writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); + } finally { + writer.close(); + } + // Get load job + Job job = writer.getJob(); + job = job.waitFor(); + LoadStatistics stats = job.getStatistics(); + return stats.getOutputRows(); + // [END writeToTable] + } + + /** + * Example of writing a local file to a table. + */ + // [TARGET writer(WriteChannelConfiguration)] + // [VARIABLE "my_dataset_name"] + // [VARIABLE "my_table_name"] + // [VARIABLE Files.newByteChannel(FileSystems.getDefault().getPath(".", "my-data.csv"))] + public long writeFileToTable(String datasetName, String tableName, ReadableByteChannel csvReader) + throws IOException, InterruptedException, TimeoutException { + // [START writeFileToTable] + TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId) .setFormatOptions(FormatOptions.csv()) @@ -342,15 +380,16 @@ public TableDataWriteChannel writeToTable(String datasetName, String tableName, TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration); // Write data to writer try { - writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); - } catch (IOException e) { - // Unable to write data + ByteStreams.copy(csvReader, writer); + } finally { + writer.close(); } - writer.close(); // Get load job Job job = writer.getJob(); - // [END writeToTable] - return writer; + job = job.waitFor(); + LoadStatistics stats = job.getStatistics(); + return stats.getOutputRows(); + // [END writeFileToTable] } /** diff --git a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITBigQuerySnippets.java b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITBigQuerySnippets.java index 38eaab709f04..03ce3831a018 100644 --- a/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITBigQuerySnippets.java +++ b/google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITBigQuerySnippets.java @@ -53,7 +53,11 @@ import org.junit.Test; import org.junit.rules.Timeout; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -177,13 +181,19 @@ public void testWriteAndListTableData() String tableName = "test_write_and_list_table_data"; String fieldName = "string_field"; assertNotNull(bigquerySnippets.createTable(DATASET, tableName, fieldName)); - TableDataWriteChannel channel = + long outputRows = bigquerySnippets.writeToTable(DATASET, tableName, "StringValue1\nStringValue2\n"); - channel.getJob().waitFor(); + assertEquals(2L, outputRows); + InputStream stream = + new ByteArrayInputStream("StringValue3\nStringValue4\n".getBytes(StandardCharsets.UTF_8)); + outputRows = bigquerySnippets.writeFileToTable(DATASET, tableName, Channels.newChannel(stream)); + assertEquals(2L, outputRows); Page> listPage = bigquerySnippets.listTableData(DATASET, tableName); Iterator> rowIterator = listPage.getValues().iterator(); assertEquals("StringValue1", rowIterator.next().get(0).getStringValue()); assertEquals("StringValue2", rowIterator.next().get(0).getStringValue()); + assertEquals("StringValue3", rowIterator.next().get(0).getStringValue()); + assertEquals("StringValue4", rowIterator.next().get(0).getStringValue()); assertTrue(bigquerySnippets.deleteTable(DATASET, tableName)); } From 614e42dada8c9418fcb8b52e3ca5202205f5a03c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 16 Nov 2016 14:21:54 -0800 Subject: [PATCH 2/2] Remove unused imports. --- .../cloud/examples/bigquery/snippets/BigQuerySnippets.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java index 3f715b03e70a..66eea5db2547 100644 --- a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java +++ b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java @@ -23,7 +23,6 @@ package com.google.cloud.examples.bigquery.snippets; import com.google.api.client.util.Charsets; -import com.google.api.services.bigquery.model.JobStatistics; import com.google.cloud.Page; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; @@ -63,9 +62,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.List;