diff --git a/bigquery/docs/snippets.py b/bigquery/docs/snippets.py index 4d7ad7506b6a..2d950936a5a6 100644 --- a/bigquery/docs/snippets.py +++ b/bigquery/docs/snippets.py @@ -1224,106 +1224,6 @@ def test_load_table_relax_column(client, to_delete): assert table.num_rows > 0 -def test_copy_table_multiple_source(client, to_delete): - dest_dataset_id = "dest_dataset_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dest_dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) - - source_dataset_id = "source_dataset_{}".format(_millis()) - source_dataset = bigquery.Dataset(client.dataset(source_dataset_id)) - source_dataset.location = "US" - source_dataset = client.create_dataset(source_dataset) - to_delete.append(source_dataset) - - schema = [ - bigquery.SchemaField("name", "STRING"), - bigquery.SchemaField("post_abbr", "STRING"), - ] - - table_data = {"table1": b"Washington,WA", "table2": b"California,CA"} - for table_id, data in table_data.items(): - table_ref = source_dataset.table(table_id) - job_config = bigquery.LoadJobConfig() - job_config.schema = schema - body = six.BytesIO(data) - client.load_table_from_file( - body, - table_ref, - # Location must match that of the destination dataset. - location="US", - job_config=job_config, - ).result() - - # [START bigquery_copy_table_multiple_source] - # from google.cloud import bigquery - # client = bigquery.Client() - # source_dataset_id = 'my_source_dataset' - # dest_dataset_id = 'my_destination_dataset' - - table1_ref = client.dataset(source_dataset_id).table("table1") - table2_ref = client.dataset(source_dataset_id).table("table2") - dest_table_ref = client.dataset(dest_dataset_id).table("destination_table") - - job = client.copy_table( - [table1_ref, table2_ref], - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - ) # API request - job.result() # Waits for job to complete. - - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) # API request - assert dest_table.num_rows > 0 - # [END bigquery_copy_table_multiple_source] - - assert dest_table.num_rows == 2 - - -def test_copy_table_cmek(client, to_delete): - dataset_id = "copy_table_cmek_{}".format(_millis()) - dest_dataset = bigquery.Dataset(client.dataset(dataset_id)) - dest_dataset.location = "US" - dest_dataset = client.create_dataset(dest_dataset) - to_delete.append(dest_dataset) - - # [START bigquery_copy_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - - source_dataset = bigquery.DatasetReference("bigquery-public-data", "samples") - source_table_ref = source_dataset.table("shakespeare") - - # dataset_id = 'my_dataset' - dest_dataset_ref = client.dataset(dataset_id) - dest_table_ref = dest_dataset_ref.table("destination_table") - - # Set the encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us", "test", "test" - ) - encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) - job_config = bigquery.CopyJobConfig() - job_config.destination_encryption_configuration = encryption_config - - job = client.copy_table( - source_table_ref, - dest_table_ref, - # Location must match that of the source and destination tables. - location="US", - job_config=job_config, - ) # API request - job.result() # Waits for job to complete. - - assert job.state == "DONE" - dest_table = client.get_table(dest_table_ref) - assert dest_table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_copy_table_cmek] - - def test_extract_table(client, to_delete): bucket_name = "extract_shakespeare_{}".format(_millis()) storage_client = storage.Client() @@ -1520,273 +1420,6 @@ def test_manage_job(client): # [END bigquery_get_job] -def test_client_query_destination_table(client, to_delete): - """Run a query""" - dataset_id = "query_destination_table_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_destination_table] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'your_dataset_id' - - job_config = bigquery.QueryJobConfig() - # Set the destination table - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - - # Start the query, passing in the extra configuration. - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query results loaded to table {}".format(table_ref.path)) - # [END bigquery_query_destination_table] - - -def test_client_query_destination_table_legacy(client, to_delete): - dataset_id = "query_destination_table_legacy_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_legacy_large_results] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_id = 'your_dataset_id' - - job_config = bigquery.QueryJobConfig() - # Set use_legacy_sql to True to use legacy SQL syntax. - job_config.use_legacy_sql = True - # Set the destination table - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - job_config.allow_large_results = True - sql = """ - SELECT corpus - FROM [bigquery-public-data:samples.shakespeare] - GROUP BY corpus; - """ - # Start the query, passing in the extra configuration. - query_job = client.query( - sql, - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query results loaded to table {}".format(table_ref.path)) - # [END bigquery_query_legacy_large_results] - - -def test_client_query_destination_table_cmek(client, to_delete): - """Run a query""" - dataset_id = "query_destination_table_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - to_delete.append(dataset_ref) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - client.create_dataset(dataset) - - # [START bigquery_query_destination_table_cmek] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - - # Set the destination table. Here, dataset_id is a string, such as: - # dataset_id = 'your_dataset_id' - table_ref = client.dataset(dataset_id).table("your_table_id") - job_config.destination = table_ref - - # Set the encryption key to use for the destination. - # TODO: Replace this key with a key you have created in KMS. - kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( - "cloud-samples-tests", "us", "test", "test" - ) - encryption_config = bigquery.EncryptionConfiguration(kms_key_name=kms_key_name) - job_config.destination_encryption_configuration = encryption_config - - # Start the query, passing in the extra configuration. - query_job = client.query( - "SELECT 17 AS my_col;", - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - query_job.result() - - # The destination table is written using the encryption configuration. - table = client.get_table(table_ref) - assert table.encryption_configuration.kms_key_name == kms_key_name - # [END bigquery_query_destination_table_cmek] - - -def test_client_query_batch(client, to_delete): - # [START bigquery_query_batch] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - # Run at batch priority, which won't count toward concurrent rate limit. - job_config.priority = bigquery.QueryPriority.BATCH - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - # Location must match that of the dataset(s) referenced in the query. - location = "US" - - # API request - starts the query - query_job = client.query(sql, location=location, job_config=job_config) - - # Check on the progress by getting the job's updated state. Once the state - # is `DONE`, the results are ready. - query_job = client.get_job( - query_job.job_id, location=location - ) # API request - fetches job - print("Job {} is currently in state {}".format(query_job.job_id, query_job.state)) - # [END bigquery_query_batch] - - -def test_client_query_relax_column(client, to_delete): - dataset_id = "query_relax_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table_ref = dataset_ref.table("my_table") - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=schema)) - - # [START bigquery_relax_column_query_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - # Retrieves the destination table and checks the number of required fields - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - # In this example, the existing table has 2 required fields - print("{} fields in the schema are required.".format(original_required_fields)) - - # Configures the query to append the results to a destination table, - # allowing field relaxation - job_config = bigquery.QueryJobConfig() - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION - ] - job_config.destination = table_ref - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - - query_job = client.query( - # In this example, the existing table contains 'full_name' and 'age' as - # required columns, but the query results will omit the second column. - 'SELECT "Beyonce" as full_name;', - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query job {} complete.".format(query_job.job_id)) - - # Checks the updated number of required fields - table = client.get_table(table) - current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) - print("{} fields in the schema are now required.".format(current_required_fields)) - # [END bigquery_relax_column_query_append] - assert original_required_fields - current_required_fields > 0 - assert len(table.schema) == 2 - assert table.schema[1].mode == "NULLABLE" - assert table.num_rows > 0 - - -def test_client_query_add_column(client, to_delete): - dataset_id = "query_add_column_{}".format(_millis()) - dataset_ref = client.dataset(dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "US" - dataset = client.create_dataset(dataset) - to_delete.append(dataset) - - table_ref = dataset_ref.table("my_table") - schema = [ - bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), - bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), - ] - table = client.create_table(bigquery.Table(table_ref, schema=schema)) - - # [START bigquery_add_column_query_append] - # from google.cloud import bigquery - # client = bigquery.Client() - # dataset_ref = client.dataset('my_dataset') - - # Retrieves the destination table and checks the length of the schema - table_id = "my_table" - table_ref = dataset_ref.table(table_id) - table = client.get_table(table_ref) - print("Table {} contains {} columns.".format(table_id, len(table.schema))) - - # Configures the query to append the results to a destination table, - # allowing field addition - job_config = bigquery.QueryJobConfig() - job_config.schema_update_options = [ - bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION - ] - job_config.destination = table_ref - job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND - - query_job = client.query( - # In this example, the existing table contains only the 'full_name' and - # 'age' columns, while the results of this query will contain an - # additional 'favorite_color' column. - 'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;', - # Location must match that of the dataset(s) referenced in the query - # and of the destination table. - location="US", - job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish - print("Query job {} complete.".format(query_job.job_id)) - - # Checks the updated length of the schema - table = client.get_table(table) - print("Table {} now contains {} columns.".format(table_id, len(table.schema))) - # [END bigquery_add_column_query_append] - assert len(table.schema) == 3 - assert table.num_rows > 0 - - def test_client_query_w_named_params(client, capsys): """Run a query using named query parameters""" @@ -1977,38 +1610,6 @@ def test_client_query_w_struct_params(client, capsys): assert "foo" in out -def test_client_query_dry_run(client): - """Run a dry run query""" - - # [START bigquery_query_dry_run] - # from google.cloud import bigquery - # client = bigquery.Client() - - job_config = bigquery.QueryJobConfig() - job_config.dry_run = True - job_config.use_query_cache = False - query_job = client.query( - ( - "SELECT name, COUNT(*) as name_count " - "FROM `bigquery-public-data.usa_names.usa_1910_2013` " - "WHERE state = 'WA' " - "GROUP BY name" - ), - # Location must match that of the dataset(s) referenced in the query. - location="US", - job_config=job_config, - ) # API request - - # A dry run query completes immediately. - assert query_job.state == "DONE" - assert query_job.dry_run - - print("This query will process {} bytes.".format(query_job.total_bytes_processed)) - # [END bigquery_query_dry_run] - - assert query_job.total_bytes_processed > 0 - - def test_query_no_cache(client): # [START bigquery_query_no_cache] # from google.cloud import bigquery diff --git a/bigquery/docs/usage/encryption.rst b/bigquery/docs/usage/encryption.rst index 88d23067995e..b512e6c4d7bf 100644 --- a/bigquery/docs/usage/encryption.rst +++ b/bigquery/docs/usage/encryption.rst @@ -36,7 +36,7 @@ Cloud KMS for the destination table. Copy a table, using a customer-managed encryption key from Cloud KMS for the destination table. -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/copy_table_cmek.py :language: python :dedent: 4 :start-after: [START bigquery_copy_table_cmek] @@ -45,7 +45,7 @@ destination table. Write query results to a table, using a customer-managed encryption key from Cloud KMS for the destination table. -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/client_query_destination_table_cmek.py :language: python :dedent: 4 :start-after: [START bigquery_query_destination_table_cmek] diff --git a/bigquery/docs/usage/queries.rst b/bigquery/docs/usage/queries.rst index 5c9dbe18fa63..9210e04bc6a2 100644 --- a/bigquery/docs/usage/queries.rst +++ b/bigquery/docs/usage/queries.rst @@ -17,7 +17,7 @@ Run a query and wait for it to finish with the Run a dry run query ^^^^^^^^^^^^^^^^^^^ -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/client_query_dry_run.py :language: python :dedent: 4 :start-after: [START bigquery_query_dry_run] @@ -30,7 +30,7 @@ Writing query results to a destination table See BigQuery documentation for more information on `writing query results `_. -.. literalinclude:: ../snippets.py +.. literalinclude:: ../samples/client_query_destination_table.py :language: python :dedent: 4 :start-after: [START bigquery_query_destination_table] diff --git a/bigquery/samples/client_query.py b/bigquery/samples/client_query.py index 9dccfd38cbcf..5242c854e220 100644 --- a/bigquery/samples/client_query.py +++ b/bigquery/samples/client_query.py @@ -30,9 +30,7 @@ def client_query(client): ORDER BY total_people DESC LIMIT 20 """ - query_job = client.query( - query, location="US" # Must match the destination dataset(s) location. - ) # Make an API request. + query_job = client.query(query) # Make an API request. print("The query data:") for row in query_job: diff --git a/bigquery/samples/client_query_add_column.py b/bigquery/samples/client_query_add_column.py new file mode 100644 index 000000000000..1cde370a35ed --- /dev/null +++ b/bigquery/samples/client_query_add_column.py @@ -0,0 +1,52 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_add_column(client, table_id): + + # [START bigquery_add_column_query_append] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the destination table. + # table_id = "your-project.your_dataset.your_table_name" + + # Retrieves the destination table and checks the length of the schema. + table = client.get_table(table_id) # Make an API request. + print("Table {} contains {} columns".format(table_id, len(table.schema))) + + # Configures the query to append the results to a destination table, + # allowing field addition. + job_config = bigquery.QueryJobConfig(destination=table_id) + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION + ] + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + + # Start the query, passing in the extra configuration. + query_job = client.query( + # In this example, the existing table contains only the 'full_name' and + # 'age' columns, while the results of this query will contain an + # additional 'favorite_color' column. + 'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;', + job_config=job_config, + ) # Make an API request. + query_job.result() # Wait for the job to complete. + + # Checks the updated length of the schema. + table = client.get_table(table_id) # Make an API request. + print("Table {} now contains {} columns".format(table_id, len(table.schema))) + # [END bigquery_add_column_query_append] diff --git a/bigquery/samples/client_query_batch.py b/bigquery/samples/client_query_batch.py new file mode 100644 index 000000000000..af9fcd8a1e40 --- /dev/null +++ b/bigquery/samples/client_query_batch.py @@ -0,0 +1,46 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_batch(client): + + # [START bigquery_query_batch] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig( + # Run at batch priority, which won't count toward concurrent rate limit. + priority=bigquery.QueryPriority.BATCH + ) + + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + + # Start the query, passing in the extra configuration. + query_job = client.query(sql, job_config=job_config) # Make an API request. + + # Check on the progress by getting the job's updated state. Once the state + # is `DONE`, the results are ready. + query_job = client.get_job( + query_job.job_id, location=query_job.location + ) # Make an API request. + + print("Job {} is currently in state {}".format(query_job.job_id, query_job.state)) + # [END bigquery_query_batch] + return query_job diff --git a/bigquery/samples/client_query_destination_table.py b/bigquery/samples/client_query_destination_table.py new file mode 100644 index 000000000000..876df7904d9c --- /dev/null +++ b/bigquery/samples/client_query_destination_table.py @@ -0,0 +1,40 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table(client, table_id): + + # [START bigquery_query_destination_table] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the destination table. + # table_id = "your-project.your_dataset.your_table_name" + + job_config = bigquery.QueryJobConfig(destination=table_id) + + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + + # Start the query, passing in the extra configuration. + query_job = client.query(sql, job_config=job_config) # Make an API request. + query_job.result() # Wait for the job to complete. + + print("Query results loaded to the table {}".format(table_id)) + # [END bigquery_query_destination_table] diff --git a/bigquery/samples/client_query_destination_table_cmek.py b/bigquery/samples/client_query_destination_table_cmek.py new file mode 100644 index 000000000000..d3409eecd77d --- /dev/null +++ b/bigquery/samples/client_query_destination_table_cmek.py @@ -0,0 +1,49 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table_cmek(client, table_id, kms_key_name): + + # [START bigquery_query_destination_table_cmek] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the destination table. + # table_id = "your-project.your_dataset.your_table_name" + + # Set the encryption key to use for the destination. + # TODO(developer): Replace this key with a key you have created in KMS. + # kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + # your-project, location, your-ring, your-key + # ) + + job_config = bigquery.QueryJobConfig( + destination=table_id, + destination_encryption_configuration=bigquery.EncryptionConfiguration( + kms_key_name=kms_key_name + ), + ) + + # Start the query, passing in the extra configuration. + query_job = client.query( + "SELECT 17 AS my_col;", job_config=job_config + ) # Make an API request. + query_job.result() # Wait for the job to complete. + + table = client.get_table(table_id) # Make an API request. + if table.encryption_configuration.kms_key_name == kms_key_name: + print("The destination table is written using the encryption configuration") + # [END bigquery_query_destination_table_cmek] diff --git a/bigquery/samples/client_query_destination_table_legacy.py b/bigquery/samples/client_query_destination_table_legacy.py new file mode 100644 index 000000000000..8e977a92d996 --- /dev/null +++ b/bigquery/samples/client_query_destination_table_legacy.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_destination_table_legacy(client, table_id): + + # [START bigquery_query_legacy_large_results] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the destination table. + # table_id = "your-project.your_dataset.your_table_name" + + # Set the destination table and use_legacy_sql to True to use + # legacy SQL syntax. + job_config = bigquery.QueryJobConfig( + allow_large_results=True, destination=table_id, use_legacy_sql=True + ) + + sql = """ + SELECT corpus + FROM [bigquery-public-data:samples.shakespeare] + GROUP BY corpus; + """ + + # Start the query, passing in the extra configuration. + query_job = client.query(sql, job_config=job_config) # Make an API request. + query_job.result() # Wait for the job to complete. + + print("Query results loaded to the table {}".format(table_id)) + # [END bigquery_query_legacy_large_results] diff --git a/bigquery/samples/client_query_dry_run.py b/bigquery/samples/client_query_dry_run.py new file mode 100644 index 000000000000..2d09a1c25f4a --- /dev/null +++ b/bigquery/samples/client_query_dry_run.py @@ -0,0 +1,40 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_dry_run(client): + + # [START bigquery_query_dry_run] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False) + + # Start the query, passing in the extra configuration. + query_job = client.query( + ( + "SELECT name, COUNT(*) as name_count " + "FROM `bigquery-public-data.usa_names.usa_1910_2013` " + "WHERE state = 'WA' " + "GROUP BY name" + ), + job_config=job_config, + ) # Make an API request. + + # A dry run query completes immediately. + print("This query will process {} bytes.".format(query_job.total_bytes_processed)) + # [END bigquery_query_dry_run] + return query_job diff --git a/bigquery/samples/client_query_legacy_sql.py b/bigquery/samples/client_query_legacy_sql.py new file mode 100644 index 000000000000..c8dae20649e2 --- /dev/null +++ b/bigquery/samples/client_query_legacy_sql.py @@ -0,0 +1,39 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_legacy_sql(client): + + # [START bigquery_query_legacy] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + query = ( + "SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] " + 'WHERE state = "TX" ' + "LIMIT 100" + ) + + # Set use_legacy_sql to True to use legacy SQL syntax. + job_config = bigquery.QueryJobConfig(use_legacy_sql=True) + + # Start the query, passing in the extra configuration. + query_job = client.query(query, job_config=job_config) # Make an API request. + + print("The query data:") + for row in query_job: + print(row) + # [END bigquery_query_legacy] diff --git a/bigquery/samples/client_query_relax_column.py b/bigquery/samples/client_query_relax_column.py new file mode 100644 index 000000000000..d8e5743c1e33 --- /dev/null +++ b/bigquery/samples/client_query_relax_column.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def client_query_relax_column(client, table_id): + + # [START bigquery_relax_column_query_append] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set table_id to the ID of the destination table. + # table_id = "your-project.your_dataset.your_table_name" + + # Retrieves the destination table and checks the number of required fields. + table = client.get_table(table_id) # Make an API request. + original_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + + # In this example, the existing table has 2 required fields. + print("{} fields in the schema are required.".format(original_required_fields)) + + # Configures the query to append the results to a destination table, + # allowing field relaxation. + job_config = bigquery.QueryJobConfig(destination=table_id) + job_config.schema_update_options = [ + bigquery.SchemaUpdateOption.ALLOW_FIELD_RELAXATION + ] + job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND + + # Start the query, passing in the extra configuration. + query_job = client.query( + # In this example, the existing table contains 'full_name' and 'age' as + # required columns, but the query results will omit the second column. + 'SELECT "Beyonce" as full_name;', + job_config=job_config, + ) # Make an API request. + query_job.result() # Wait for the job to complete. + + # Checks the updated number of required fields. + table = client.get_table(table_id) # Make an API request. + current_required_fields = sum(field.mode == "REQUIRED" for field in table.schema) + print("{} fields in the schema are now required.".format(current_required_fields)) + # [END bigquery_relax_column_query_append] diff --git a/bigquery/samples/copy_table.py b/bigquery/samples/copy_table.py index f6ebd91470eb..20f6776cf87d 100644 --- a/bigquery/samples/copy_table.py +++ b/bigquery/samples/copy_table.py @@ -28,12 +28,8 @@ def copy_table(client, source_table_id, destination_table_id): # TODO(developer): Set destination_table_id to the ID of the destination table. # destination_table_id = "your-project.destination_dataset.destination_table" - job = client.copy_table( - source_table_id, - destination_table_id, - location="US", # Must match the source and destination tables location. - ) - job.result() # Waits for job to complete. + job = client.copy_table(source_table_id, destination_table_id) + job.result() # Wait for the job to complete. print("A copy of the table created.") # [END bigquery_copy_table] diff --git a/bigquery/samples/copy_table_cmek.py b/bigquery/samples/copy_table_cmek.py new file mode 100644 index 000000000000..1e9ee198c821 --- /dev/null +++ b/bigquery/samples/copy_table_cmek.py @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def copy_table_cmek(client, dest_table_id, orig_table_id, kms_key_name): + + # [START bigquery_copy_table_cmek] + from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dest_table_id to the ID of the destination table. + # dest_table_id = "your-project.your_dataset.your_table_name" + + # TODO(developer): Set orig_table_id to the ID of the original table. + # orig_table_id = "your-project.your_dataset.your_table_name" + + # Set the encryption key to use for the destination. + # TODO(developer): Replace this key with a key you have created in KMS. + # kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format( + # your-project, location, your-ring, your-key + # ) + + job_config = bigquery.CopyJobConfig( + destination_encryption_configuration=bigquery.EncryptionConfiguration( + kms_key_name=kms_key_name + ) + ) + job = client.copy_table(orig_table_id, dest_table_id, job_config=job_config) + job.result() # Wait for the job to complete. + + dest_table = client.get_table(dest_table_id) # Make an API request. + if dest_table.encryption_configuration.kms_key_name == kms_key_name: + print("A copy of the table created") + # [END bigquery_copy_table_cmek] diff --git a/bigquery/samples/copy_table_multiple_source.py b/bigquery/samples/copy_table_multiple_source.py new file mode 100644 index 000000000000..532ea0a0ab90 --- /dev/null +++ b/bigquery/samples/copy_table_multiple_source.py @@ -0,0 +1,35 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def copy_table_multiple_source(client, dest_table_id, table_ids): + + # [START bigquery_copy_table_multiple_source] + # TODO(developer): Import the client library. + # from google.cloud import bigquery + + # TODO(developer): Construct a BigQuery client object. + # client = bigquery.Client() + + # TODO(developer): Set dest_table_id to the ID of the destination table. + # dest_table_id = "your-project.your_dataset.your_table_name" + + # TODO(developer): Set table_ids to the list of the IDs of the original tables. + # table_ids = ["your-project.your_dataset.your_table_name", ...] + + job = client.copy_table(table_ids, dest_table_id) # Make an API request. + job.result() # Wait for the job to complete. + + print("The tables {} have been appended to {}".format(table_ids, dest_table_id)) + # [END bigquery_copy_table_multiple_source] diff --git a/bigquery/samples/load_table_dataframe.py b/bigquery/samples/load_table_dataframe.py index ea6fe5d02384..8cfb34424457 100644 --- a/bigquery/samples/load_table_dataframe.py +++ b/bigquery/samples/load_table_dataframe.py @@ -61,10 +61,7 @@ def load_table_dataframe(client, table_id): ) job = client.load_table_from_dataframe( - dataframe, - table_id, - job_config=job_config, - location="US", # Must match the destination dataset location. + dataframe, table_id, job_config=job_config ) # Make an API request. job.result() # Wait for the job to complete. diff --git a/bigquery/samples/query_external_sheets_permanent_table.py b/bigquery/samples/query_external_sheets_permanent_table.py index ce9b1c928782..e9bc908f5b15 100644 --- a/bigquery/samples/query_external_sheets_permanent_table.py +++ b/bigquery/samples/query_external_sheets_permanent_table.py @@ -61,6 +61,7 @@ def query_external_sheets_permanent_table(dataset_id): # Example query to find states starting with "W". sql = 'SELECT * FROM `{}.{}` WHERE name LIKE "W%"'.format(dataset_id, table_id) + query_job = client.query(sql) # Make an API request. # Wait for the query to complete. diff --git a/bigquery/samples/query_external_sheets_temporary_table.py b/bigquery/samples/query_external_sheets_temporary_table.py index e89b6efab362..7b6bde864b09 100644 --- a/bigquery/samples/query_external_sheets_temporary_table.py +++ b/bigquery/samples/query_external_sheets_temporary_table.py @@ -52,11 +52,11 @@ def query_external_sheets_temporary_table(): "us-states!A20:B49" # Optionally set range of the sheet to query from. ) table_id = "us_states" - job_config = bigquery.QueryJobConfig() - job_config.table_definitions = {table_id: external_config} + job_config = bigquery.QueryJobConfig(table_definitions={table_id: external_config}) # Example query to find states starting with "W". sql = 'SELECT * FROM `{}` WHERE name LIKE "W%"'.format(table_id) + query_job = client.query(sql, job_config=job_config) # Make an API request. # Wait for the query to complete. diff --git a/bigquery/samples/tests/conftest.py b/bigquery/samples/tests/conftest.py index a06bb9c90d1d..6d049e6c2312 100644 --- a/bigquery/samples/tests/conftest.py +++ b/bigquery/samples/tests/conftest.py @@ -148,3 +148,8 @@ def model_id(client, dataset_id): client.query(sql).result() return model_id + + +@pytest.fixture +def kms_key_name(): + return "projects/cloud-samples-tests/locations/us/keyRings/test/cryptoKeys/test" diff --git a/bigquery/samples/tests/test_add_empty_column.py b/bigquery/samples/tests/test_add_empty_column.py index e6c56e6cbfbc..de51bfed7672 100644 --- a/bigquery/samples/tests/test_add_empty_column.py +++ b/bigquery/samples/tests/test_add_empty_column.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import add_empty_column diff --git a/bigquery/samples/tests/test_browse_table_data.py b/bigquery/samples/tests/test_browse_table_data.py index 0e9cc6055494..db9b867f5ab7 100644 --- a/bigquery/samples/tests/test_browse_table_data.py +++ b/bigquery/samples/tests/test_browse_table_data.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import browse_table_data diff --git a/bigquery/samples/tests/test_client_list_jobs.py b/bigquery/samples/tests/test_client_list_jobs.py index 011e081fdee4..ada053239802 100644 --- a/bigquery/samples/tests/test_client_list_jobs.py +++ b/bigquery/samples/tests/test_client_list_jobs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import client_list_jobs from .. import create_job diff --git a/bigquery/samples/tests/test_client_query.py b/bigquery/samples/tests/test_client_query.py index fd5b8e7edd97..e73e7e5a0eb4 100644 --- a/bigquery/samples/tests/test_client_query.py +++ b/bigquery/samples/tests/test_client_query.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import client_query diff --git a/bigquery/samples/tests/test_client_query_add_column.py b/bigquery/samples/tests/test_client_query_add_column.py new file mode 100644 index 000000000000..67ac328d5518 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_add_column.py @@ -0,0 +1,32 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery + +from .. import client_query_add_column + + +def test_client_query_add_column(capsys, client, random_table_id): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + client.create_table(bigquery.Table(random_table_id, schema=schema)) + + client_query_add_column.client_query_add_column(client, random_table_id) + out, err = capsys.readouterr() + assert "Table {} contains 2 columns".format(random_table_id) in out + assert "Table {} now contains 3 columns".format(random_table_id) in out diff --git a/bigquery/samples/tests/test_create_routine.py b/bigquery/samples/tests/test_client_query_batch.py similarity index 73% rename from bigquery/samples/tests/test_create_routine.py rename to bigquery/samples/tests/test_client_query_batch.py index 7220d63542e2..79197e4565c7 100644 --- a/bigquery/samples/tests/test_create_routine.py +++ b/bigquery/samples/tests/test_client_query_batch.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .. import client_query_batch -from .. import create_routine +def test_client_query_batch(capsys, client): -def test_create_routine(capsys, client, random_routine_id): - - create_routine.create_routine(client, random_routine_id) + job = client_query_batch.client_query_batch(client) out, err = capsys.readouterr() - assert "Created routine {}".format(random_routine_id) in out + assert "Job {} is currently in state {}".format(job.job_id, job.state) in out diff --git a/bigquery/samples/tests/test_list_routines.py b/bigquery/samples/tests/test_client_query_destination_table.py similarity index 69% rename from bigquery/samples/tests/test_list_routines.py rename to bigquery/samples/tests/test_client_query_destination_table.py index e249238e1976..d29aaebd3ce5 100644 --- a/bigquery/samples/tests/test_list_routines.py +++ b/bigquery/samples/tests/test_client_query_destination_table.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .. import client_query_destination_table -from .. import list_routines +def test_client_query_destination_table(capsys, client, table_id): -def test_list_routines(capsys, client, dataset_id, routine_id): - - list_routines.list_routines(client, dataset_id) + client_query_destination_table.client_query_destination_table(client, table_id) out, err = capsys.readouterr() - assert "Routines contained in dataset {}:".format(dataset_id) in out - assert routine_id in out + assert "Query results loaded to the table {}".format(table_id) in out diff --git a/bigquery/samples/tests/test_client_query_destination_table_cmek.py b/bigquery/samples/tests/test_client_query_destination_table_cmek.py new file mode 100644 index 000000000000..cd4532be6d1d --- /dev/null +++ b/bigquery/samples/tests/test_client_query_destination_table_cmek.py @@ -0,0 +1,26 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import client_query_destination_table_cmek + + +def test_client_query_destination_table_cmek( + capsys, client, random_table_id, kms_key_name +): + + client_query_destination_table_cmek.client_query_destination_table_cmek( + client, random_table_id, kms_key_name + ) + out, err = capsys.readouterr() + assert "The destination table is written using the encryption configuration" in out diff --git a/bigquery/samples/tests/test_get_routine.py b/bigquery/samples/tests/test_client_query_destination_table_legacy.py similarity index 64% rename from bigquery/samples/tests/test_get_routine.py rename to bigquery/samples/tests/test_client_query_destination_table_legacy.py index fa5f3093116c..da62baada213 100644 --- a/bigquery/samples/tests/test_get_routine.py +++ b/bigquery/samples/tests/test_client_query_destination_table_legacy.py @@ -12,16 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .. import client_query_destination_table_legacy -from .. import get_routine +def test_client_query_destination_table_legacy(capsys, client, random_table_id): -def test_get_routine(capsys, client, routine_id): - - get_routine.get_routine(client, routine_id) + client_query_destination_table_legacy.client_query_destination_table_legacy( + client, random_table_id + ) out, err = capsys.readouterr() - assert "Routine '{}':".format(routine_id) in out - assert "Type: 'SCALAR_FUNCTION'" in out - assert "Language: 'SQL'" in out - assert "Name: 'x'" in out - assert "Type: 'type_kind: INT64\n'" in out + assert "Query results loaded to the table {}".format(random_table_id) in out diff --git a/bigquery/samples/tests/test_client_query_dry_run.py b/bigquery/samples/tests/test_client_query_dry_run.py new file mode 100644 index 000000000000..c39a22767d25 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_dry_run.py @@ -0,0 +1,25 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import client_query_dry_run + + +def test_client_query_dry_run(capsys, client): + + query_job = client_query_dry_run.client_query_dry_run(client) + out, err = capsys.readouterr() + assert "This query will process" in out + assert query_job.state == "DONE" + assert query_job.dry_run + assert query_job.total_bytes_processed > 0 diff --git a/bigquery/samples/tests/test_delete_routine.py b/bigquery/samples/tests/test_client_query_legacy_sql.py similarity index 74% rename from bigquery/samples/tests/test_delete_routine.py rename to bigquery/samples/tests/test_client_query_legacy_sql.py index 9347d1e22dc2..fb6ee60bc6ec 100644 --- a/bigquery/samples/tests/test_delete_routine.py +++ b/bigquery/samples/tests/test_client_query_legacy_sql.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re -from .. import delete_routine +from .. import client_query_legacy_sql -def test_delete_routine(capsys, client, routine_id): +def test_client_query_legacy_sql(capsys, client): - delete_routine.delete_routine(client, routine_id) + client_query_legacy_sql.client_query_legacy_sql(client) out, err = capsys.readouterr() - assert "Deleted routine {}.".format(routine_id) in out + assert re.search(r"(Row[\w(){}:', ]+)$", out) diff --git a/bigquery/samples/tests/test_client_query_relax_column.py b/bigquery/samples/tests/test_client_query_relax_column.py new file mode 100644 index 000000000000..685db9cb1fa0 --- /dev/null +++ b/bigquery/samples/tests/test_client_query_relax_column.py @@ -0,0 +1,32 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery + +from .. import client_query_relax_column + + +def test_client_query_relax_column(capsys, client, random_table_id): + + schema = [ + bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"), + bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"), + ] + + client.create_table(bigquery.Table(random_table_id, schema=schema)) + + client_query_relax_column.client_query_relax_column(client, random_table_id) + out, err = capsys.readouterr() + assert "2 fields in the schema are required." in out + assert "0 fields in the schema are now required." in out diff --git a/bigquery/samples/tests/test_copy_table.py b/bigquery/samples/tests/test_copy_table.py index 6d7de2d9132c..0138cd8ee1e2 100644 --- a/bigquery/samples/tests/test_copy_table.py +++ b/bigquery/samples/tests/test_copy_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import copy_table diff --git a/bigquery/samples/tests/test_copy_table_cmek.py b/bigquery/samples/tests/test_copy_table_cmek.py new file mode 100644 index 000000000000..25238071b947 --- /dev/null +++ b/bigquery/samples/tests/test_copy_table_cmek.py @@ -0,0 +1,26 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import copy_table_cmek + + +def test_copy_table_cmek( + capsys, client, random_table_id, table_with_data_id, kms_key_name +): + + copy_table_cmek.copy_table_cmek( + client, random_table_id, table_with_data_id, kms_key_name + ) + out, err = capsys.readouterr() + assert "A copy of the table created" in out diff --git a/bigquery/samples/tests/test_copy_table_multiple_source.py b/bigquery/samples/tests/test_copy_table_multiple_source.py new file mode 100644 index 000000000000..755fa2ccb5e9 --- /dev/null +++ b/bigquery/samples/tests/test_copy_table_multiple_source.py @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +from google.cloud import bigquery + +from .. import copy_table_multiple_source + + +def test_copy_table_multiple_source(capsys, client, random_table_id, random_dataset_id): + + schema = [ + bigquery.SchemaField("name", "STRING"), + bigquery.SchemaField("post_abbr", "STRING"), + ] + + dataset = bigquery.Dataset(random_dataset_id) + dataset.location = "US" + dataset = client.create_dataset(dataset) + table_data = {"table1": b"Washington,WA", "table2": b"California,CA"} + for table_id, data in table_data.items(): + table_ref = dataset.table(table_id) + job_config = bigquery.LoadJobConfig() + job_config.schema = schema + body = six.BytesIO(data) + client.load_table_from_file( + body, table_ref, location="US", job_config=job_config + ).result() + + table_ids = [ + "{}.table1".format(random_dataset_id), + "{}.table2".format(random_dataset_id), + ] + + copy_table_multiple_source.copy_table_multiple_source( + client, random_table_id, table_ids + ) + dest_table = client.get_table(random_table_id) + out, err = capsys.readouterr() + assert ( + "The tables {} have been appended to {}".format(table_ids, random_table_id) + in out + ) + assert dest_table.num_rows > 0 diff --git a/bigquery/samples/tests/test_create_dataset.py b/bigquery/samples/tests/test_create_dataset.py index e52e9ddfdced..dfadc67d8468 100644 --- a/bigquery/samples/tests/test_create_dataset.py +++ b/bigquery/samples/tests/test_create_dataset.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import create_dataset diff --git a/bigquery/samples/tests/test_create_job.py b/bigquery/samples/tests/test_create_job.py index 5ead51156606..bbf880cbe402 100644 --- a/bigquery/samples/tests/test_create_job.py +++ b/bigquery/samples/tests/test_create_job.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import create_job def test_create_job(capsys, client): query_job = create_job.create_job(client) - client.cancel_job(query_job.job_id, location="US") + client.cancel_job(query_job.job_id, location=query_job.location) out, err = capsys.readouterr() assert "Started job: {}".format(query_job.job_id) in out diff --git a/bigquery/samples/tests/test_create_table.py b/bigquery/samples/tests/test_create_table.py index f9ebc0e5d70d..093ee6e94277 100644 --- a/bigquery/samples/tests/test_create_table.py +++ b/bigquery/samples/tests/test_create_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import create_table diff --git a/bigquery/samples/tests/test_create_table_range_partitioned.py b/bigquery/samples/tests/test_create_table_range_partitioned.py index ca186f9a7554..ac312b033832 100644 --- a/bigquery/samples/tests/test_create_table_range_partitioned.py +++ b/bigquery/samples/tests/test_create_table_range_partitioned.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import create_table_range_partitioned diff --git a/bigquery/samples/tests/test_dataset_exists.py b/bigquery/samples/tests/test_dataset_exists.py index 203c4b884dc4..a44e60371120 100644 --- a/bigquery/samples/tests/test_dataset_exists.py +++ b/bigquery/samples/tests/test_dataset_exists.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from .. import dataset_exists diff --git a/bigquery/samples/tests/test_dataset_label_samples.py b/bigquery/samples/tests/test_dataset_label_samples.py index 1e526f2339ac..94a2092407b0 100644 --- a/bigquery/samples/tests/test_dataset_label_samples.py +++ b/bigquery/samples/tests/test_dataset_label_samples.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import delete_dataset_labels from .. import get_dataset_labels from .. import label_dataset diff --git a/bigquery/samples/tests/test_delete_dataset.py b/bigquery/samples/tests/test_delete_dataset.py index 836b3aebb272..2b1b6ad06195 100644 --- a/bigquery/samples/tests/test_delete_dataset.py +++ b/bigquery/samples/tests/test_delete_dataset.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import delete_dataset diff --git a/bigquery/samples/tests/test_delete_table.py b/bigquery/samples/tests/test_delete_table.py index f76ad8624cc6..8f4796623a83 100644 --- a/bigquery/samples/tests/test_delete_table.py +++ b/bigquery/samples/tests/test_delete_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import delete_table diff --git a/bigquery/samples/tests/test_get_dataset.py b/bigquery/samples/tests/test_get_dataset.py index 8682be7ee3e9..dedec1d7b29e 100644 --- a/bigquery/samples/tests/test_get_dataset.py +++ b/bigquery/samples/tests/test_get_dataset.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import get_dataset diff --git a/bigquery/samples/tests/test_get_table.py b/bigquery/samples/tests/test_get_table.py index 8adaa6557954..b950d434aef6 100644 --- a/bigquery/samples/tests/test_get_table.py +++ b/bigquery/samples/tests/test_get_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from .. import get_table diff --git a/bigquery/samples/tests/test_list_datasets.py b/bigquery/samples/tests/test_list_datasets.py index d8c32e91ee20..4c66a24f9b1a 100644 --- a/bigquery/samples/tests/test_list_datasets.py +++ b/bigquery/samples/tests/test_list_datasets.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import list_datasets diff --git a/bigquery/samples/tests/test_list_datasets_by_label.py b/bigquery/samples/tests/test_list_datasets_by_label.py index f414539b00b3..6d04a281ff42 100644 --- a/bigquery/samples/tests/test_list_datasets_by_label.py +++ b/bigquery/samples/tests/test_list_datasets_by_label.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import list_datasets_by_label diff --git a/bigquery/samples/tests/test_list_tables.py b/bigquery/samples/tests/test_list_tables.py index 61ac04ea26ce..ec1621ac7579 100644 --- a/bigquery/samples/tests/test_list_tables.py +++ b/bigquery/samples/tests/test_list_tables.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import list_tables diff --git a/bigquery/samples/tests/test_load_table_dataframe.py b/bigquery/samples/tests/test_load_table_dataframe.py index 2151704d3b25..3b7cb16ea692 100644 --- a/bigquery/samples/tests/test_load_table_dataframe.py +++ b/bigquery/samples/tests/test_load_table_dataframe.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import pytest from .. import load_table_dataframe diff --git a/bigquery/samples/tests/test_model_samples.py b/bigquery/samples/tests/test_model_samples.py index 99d838533917..d7b06a92a3e1 100644 --- a/bigquery/samples/tests/test_model_samples.py +++ b/bigquery/samples/tests/test_model_samples.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import delete_model from .. import get_model from .. import list_models diff --git a/bigquery/samples/tests/test_query_external_sheets_permanent_table.py b/bigquery/samples/tests/test_query_external_sheets_permanent_table.py index a7b5db09e5af..a00930cad881 100644 --- a/bigquery/samples/tests/test_query_external_sheets_permanent_table.py +++ b/bigquery/samples/tests/test_query_external_sheets_permanent_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import query_external_sheets_permanent_table diff --git a/bigquery/samples/tests/test_query_external_sheets_temporary_table.py b/bigquery/samples/tests/test_query_external_sheets_temporary_table.py index 4856b6a49d2b..8274787cb644 100644 --- a/bigquery/samples/tests/test_query_external_sheets_temporary_table.py +++ b/bigquery/samples/tests/test_query_external_sheets_temporary_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import query_external_sheets_temporary_table diff --git a/bigquery/samples/tests/test_query_to_arrow.py b/bigquery/samples/tests/test_query_to_arrow.py index 2fbed807ece4..dd9b3ab508cc 100644 --- a/bigquery/samples/tests/test_query_to_arrow.py +++ b/bigquery/samples/tests/test_query_to_arrow.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import pyarrow from .. import query_to_arrow diff --git a/bigquery/samples/tests/test_create_routine_ddl.py b/bigquery/samples/tests/test_routine_samples.py similarity index 62% rename from bigquery/samples/tests/test_create_routine_ddl.py rename to bigquery/samples/tests/test_routine_samples.py index bcb3249d26ef..81d33a0cf5df 100644 --- a/bigquery/samples/tests/test_create_routine_ddl.py +++ b/bigquery/samples/tests/test_routine_samples.py @@ -12,19 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from google.cloud import bigquery_v2 -from .. import create_routine_ddl + +def test_create_routine(capsys, client, random_routine_id): + from .. import create_routine + + create_routine.create_routine(client, random_routine_id) + out, err = capsys.readouterr() + assert "Created routine {}".format(random_routine_id) in out def test_create_routine_ddl(capsys, client, random_routine_id): + from .. import create_routine_ddl create_routine_ddl.create_routine_ddl(client, random_routine_id) routine = client.get_routine(random_routine_id) out, err = capsys.readouterr() + assert "Created routine {}".format(random_routine_id) in out + return routine assert routine.type_ == "SCALAR_FUNCTION" assert routine.language == "SQL" expected_arguments = [ @@ -55,3 +63,39 @@ def test_create_routine_ddl(capsys, client, random_routine_id): ) ] assert routine.arguments == expected_arguments + + +def test_list_routines(capsys, client, dataset_id, routine_id): + from .. import list_routines + + list_routines.list_routines(client, dataset_id) + out, err = capsys.readouterr() + assert "Routines contained in dataset {}:".format(dataset_id) in out + assert routine_id in out + + +def test_get_routine(capsys, client, routine_id): + from .. import get_routine + + get_routine.get_routine(client, routine_id) + out, err = capsys.readouterr() + assert "Routine '{}':".format(routine_id) in out + assert "Type: 'SCALAR_FUNCTION'" in out + assert "Language: 'SQL'" in out + assert "Name: 'x'" in out + assert "Type: 'type_kind: INT64\n'" in out + + +def test_delete_routine(capsys, client, routine_id): + from .. import delete_routine + + delete_routine.delete_routine(client, routine_id) + out, err = capsys.readouterr() + assert "Deleted routine {}.".format(routine_id) in out + + +def test_update_routine(client, routine_id): + from .. import update_routine + + routine = update_routine.update_routine(client, routine_id) + assert routine.body == "x * 4" diff --git a/bigquery/samples/tests/test_table_exists.py b/bigquery/samples/tests/test_table_exists.py index 232d77fbcb60..ae4fc65f847c 100644 --- a/bigquery/samples/tests/test_table_exists.py +++ b/bigquery/samples/tests/test_table_exists.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from .. import table_exists diff --git a/bigquery/samples/tests/test_table_insert_rows.py b/bigquery/samples/tests/test_table_insert_rows.py index 95d119dbdc93..9c5fd5768cfb 100644 --- a/bigquery/samples/tests/test_table_insert_rows.py +++ b/bigquery/samples/tests/test_table_insert_rows.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from .. import table_insert_rows diff --git a/bigquery/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py b/bigquery/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py index 6a59609baacf..a2a4febd7f75 100644 --- a/bigquery/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py +++ b/bigquery/samples/tests/test_table_insert_rows_explicit_none_insert_ids.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from google.cloud import bigquery from .. import table_insert_rows_explicit_none_insert_ids as mut diff --git a/bigquery/samples/tests/test_update_dataset_access.py b/bigquery/samples/tests/test_update_dataset_access.py index 679b700731e3..ae33dbfe4a4c 100644 --- a/bigquery/samples/tests/test_update_dataset_access.py +++ b/bigquery/samples/tests/test_update_dataset_access.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import update_dataset_access diff --git a/bigquery/samples/tests/test_update_dataset_default_table_expiration.py b/bigquery/samples/tests/test_update_dataset_default_table_expiration.py index a97de11a2f1a..46e9654209ed 100644 --- a/bigquery/samples/tests/test_update_dataset_default_table_expiration.py +++ b/bigquery/samples/tests/test_update_dataset_default_table_expiration.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import update_dataset_default_table_expiration diff --git a/bigquery/samples/tests/test_update_dataset_description.py b/bigquery/samples/tests/test_update_dataset_description.py index 63826077b976..c6f8889f50da 100644 --- a/bigquery/samples/tests/test_update_dataset_description.py +++ b/bigquery/samples/tests/test_update_dataset_description.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - from .. import update_dataset_description diff --git a/bigquery/samples/tests/test_update_routine.py b/bigquery/samples/tests/test_update_routine.py deleted file mode 100644 index 8adfab32e032..000000000000 --- a/bigquery/samples/tests/test_update_routine.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from .. import update_routine - - -def test_update_routine(client, routine_id): - - routine = update_routine.update_routine(client, routine_id) - assert routine.body == "x * 4" diff --git a/bigquery/samples/undelete_table.py b/bigquery/samples/undelete_table.py index 2d544cf5aa8c..9db9712d2a74 100644 --- a/bigquery/samples/undelete_table.py +++ b/bigquery/samples/undelete_table.py @@ -46,7 +46,7 @@ def undelete_table(client, table_id, recovered_table_id): # [END_EXCLUDE] # "Accidentally" delete the table. - client.delete_table(table_id) # API request + client.delete_table(table_id) # Make an API request. # Construct the restore-from table ID using a snapshot decorator. snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch) @@ -55,11 +55,11 @@ def undelete_table(client, table_id, recovered_table_id): job = client.copy_table( snapshot_table_id, recovered_table_id, - # Location must match that of the source and destination tables. + # Must match the source and destination tables location. location="US", - ) # API request + ) # Make an API request. - job.result() # Wait for job to complete. + job.result() # Wait for the job to complete. print( "Copied data from deleted table {} to {}".format(table_id, recovered_table_id)