diff --git a/bigtable/docs/snippets.py b/bigtable/docs/snippets.py index 3e80d09c667f..fa67220ad9b7 100644 --- a/bigtable/docs/snippets.py +++ b/bigtable/docs/snippets.py @@ -33,14 +33,17 @@ import pytest from test_utils.system import unique_resource_id +from test_utils.retry import RetryErrors from google.api_core.exceptions import NotFound +from google.api_core.exceptions import TooManyRequests from google.cloud._helpers import UTC from google.cloud.bigtable import Client from google.cloud.bigtable import enums -INSTANCE_ID = "snippet-tests" + unique_resource_id("-") -CLUSTER_ID = "clus-1-" + unique_resource_id("-") +UNIQUE_SUFFIX = unique_resource_id("-") +INSTANCE_ID = "snippet-tests" + UNIQUE_SUFFIX +CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX LOCATION_ID = "us-central1-f" ALT_LOCATION_ID = "us-central1-a" PRODUCTION = enums.Instance.Type.PRODUCTION @@ -55,6 +58,8 @@ LABELS = {LABEL_KEY: str(LABEL_STAMP)} INSTANCES_TO_DELETE = [] +retry_429 = RetryErrors(TooManyRequests, max_tries=9) + class Config(object): """Run-time configuration to be modified at set-up. @@ -81,13 +86,14 @@ def setup_module(): operation = Config.INSTANCE.create(clusters=[cluster]) # We want to make sure the operation completes. operation.result(timeout=100) - INSTANCES_TO_DELETE.append(Config.INSTANCE) def teardown_module(): + retry_429(Config.INSTANCE.delete)() + for instance in INSTANCES_TO_DELETE: try: - instance.delete() + retry_429(instance.delete)() except NotFound: pass @@ -97,8 +103,8 @@ def test_bigtable_create_instance(): from google.cloud.bigtable import Client from google.cloud.bigtable import enums - my_instance_id = "inst-my-" + unique_resource_id("-") - my_cluster_id = "clus-my-" + unique_resource_id("-") + my_instance_id = "inst-my-" + UNIQUE_SUFFIX + my_cluster_id = "clus-my-" + UNIQUE_SUFFIX location_id = "us-central1-f" serve_nodes = 3 storage_type = enums.StorageType.SSD @@ -115,15 +121,15 @@ def test_bigtable_create_instance(): ) operation = instance.create(clusters=[cluster]) - # Make sure this instance gets deleted after the test case. - INSTANCES_TO_DELETE.append(instance) - # We want to make sure the operation completes. operation.result(timeout=100) + # [END bigtable_create_prod_instance] - assert instance.exists() - instance.delete() + try: + assert instance.exists() + finally: + retry_429(instance.delete)() def test_bigtable_create_additional_cluster(): @@ -139,7 +145,7 @@ def test_bigtable_create_additional_cluster(): client = Client(admin=True) instance = client.instance(INSTANCE_ID) - cluster_id = "clus-my-" + unique_resource_id("-") + cluster_id = "clus-my-" + UNIQUE_SUFFIX location_id = "us-central1-a" serve_nodes = 3 storage_type = enums.StorageType.SSD @@ -154,9 +160,11 @@ def test_bigtable_create_additional_cluster(): # We want to make sure the operation completes. operation.result(timeout=100) # [END bigtable_create_cluster] - assert cluster.exists() - cluster.delete() + try: + assert cluster.exists() + finally: + retry_429(cluster.delete)() def test_bigtable_create_app_profile(): @@ -166,7 +174,7 @@ def test_bigtable_create_app_profile(): client = Client(admin=True) instance = client.instance(INSTANCE_ID) - app_profile_id = "app-prof-" + unique_resource_id("-") + app_profile_id = "app-prof-" + UNIQUE_SUFFIX description = "routing policy-multy" routing_policy_type = enums.RoutingPolicyType.ANY @@ -179,9 +187,11 @@ def test_bigtable_create_app_profile(): app_profile = app_profile.create(ignore_warnings=True) # [END bigtable_create_app_profile] - assert app_profile.exists() - app_profile.delete(ignore_warnings=True) + try: + assert app_profile.exists() + finally: + retry_429(app_profile.delete)(ignore_warnings=True) def test_bigtable_list_instances(): @@ -191,6 +201,7 @@ def test_bigtable_list_instances(): client = Client(admin=True) (instances_list, failed_locations_list) = client.list_instances() # [END bigtable_list_instances] + assert len(instances_list) > 0 @@ -202,6 +213,7 @@ def test_bigtable_list_clusters_on_instance(): instance = client.instance(INSTANCE_ID) (clusters_list, failed_locations_list) = instance.list_clusters() # [END bigtable_list_clusters_on_instance] + assert len(clusters_list) > 0 @@ -212,12 +224,13 @@ def test_bigtable_list_clusters_in_project(): client = Client(admin=True) (clusters_list, failed_locations_list) = client.list_clusters() # [END bigtable_list_clusters_in_project] + assert len(clusters_list) > 0 def test_bigtable_list_app_profiles(): app_profile = Config.INSTANCE.app_profile( - app_profile_id="app-prof-" + unique_resource_id("-"), + app_profile_id="app-prof-" + UNIQUE_SUFFIX, routing_policy_type=enums.RoutingPolicyType.ANY, ) app_profile = app_profile.create(ignore_warnings=True) @@ -230,7 +243,11 @@ def test_bigtable_list_app_profiles(): app_profiles_list = instance.list_app_profiles() # [END bigtable_list_app_profiles] - assert len(app_profiles_list) > 0 + + try: + assert len(app_profiles_list) > 0 + finally: + retry_429(app_profile.delete)(ignore_warnings=True) def test_bigtable_instance_exists(): @@ -241,6 +258,7 @@ def test_bigtable_instance_exists(): instance = client.instance(INSTANCE_ID) instance_exists = instance.exists() # [END bigtable_check_instance_exists] + assert instance_exists @@ -253,6 +271,7 @@ def test_bigtable_cluster_exists(): cluster = instance.cluster(CLUSTER_ID) cluster_exists = cluster.exists() # [END bigtable_check_cluster_exists] + assert cluster_exists @@ -264,6 +283,7 @@ def test_bigtable_reload_instance(): instance = client.instance(INSTANCE_ID) instance.reload() # [END bigtable_reload_instance] + assert instance.type_ == PRODUCTION.value @@ -276,6 +296,7 @@ def test_bigtable_reload_cluster(): cluster = instance.cluster(CLUSTER_ID) cluster.reload() # [END bigtable_reload_cluster] + assert cluster.serve_nodes == SERVER_NODES @@ -289,10 +310,8 @@ def test_bigtable_update_instance(): instance.display_name = display_name instance.update() # [END bigtable_update_instance] - assert instance.display_name == display_name - # Make sure this instance gets deleted after the test case. - INSTANCES_TO_DELETE.append(instance) + assert instance.display_name == display_name def test_bigtable_update_cluster(): @@ -305,6 +324,7 @@ def test_bigtable_update_cluster(): cluster.serve_nodes = 4 cluster.update() # [END bigtable_update_cluster] + assert cluster.serve_nodes == 4 @@ -320,10 +340,23 @@ def test_bigtable_create_table(): max_versions_rule = column_family.MaxVersionsGCRule(2) table.create(column_families={"cf1": max_versions_rule}) # [END bigtable_create_table] - assert table.exists() + + try: + assert table.exists() + finally: + retry_429(table.delete)() def test_bigtable_list_tables(): + from google.cloud.bigtable import Client + from google.cloud.bigtable import column_family + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + table = instance.table("to_list") + max_versions_rule = column_family.MaxVersionsGCRule(2) + table.create(column_families={"cf1": max_versions_rule}) + # [START bigtable_list_tables] from google.cloud.bigtable import Client @@ -331,7 +364,12 @@ def test_bigtable_list_tables(): instance = client.instance(INSTANCE_ID) tables_list = instance.list_tables() # [END bigtable_list_tables] - assert len(tables_list) > 0 + + table_names = [table.name for table in tables_list] + try: + assert table.name in table_names + finally: + retry_429(table.delete)() def test_bigtable_delete_cluster(): @@ -339,7 +377,7 @@ def test_bigtable_delete_cluster(): client = Client(admin=True) instance = client.instance(INSTANCE_ID) - cluster_id = "clus-my-" + unique_resource_id("-") + cluster_id = "clus-my-" + UNIQUE_SUFFIX cluster = instance.cluster( cluster_id, location_id=ALT_LOCATION_ID, @@ -359,6 +397,7 @@ def test_bigtable_delete_cluster(): cluster_to_delete.delete() # [END bigtable_delete_cluster] + assert not cluster_to_delete.exists() @@ -376,12 +415,13 @@ def test_bigtable_delete_instance(): ) operation = instance.create(clusters=[cluster]) - # Make sure this instance gets deleted after the test case. - INSTANCES_TO_DELETE.append(instance) # We want to make sure the operation completes. operation.result(timeout=100) + # Make sure this instance gets deleted after the test case. + INSTANCES_TO_DELETE.append(instance) + # [START bigtable_delete_instance] from google.cloud.bigtable import Client @@ -394,6 +434,9 @@ def test_bigtable_delete_instance(): assert not instance_to_delete.exists() + # Skip deleting it during module teardown if the assertion succeeds. + INSTANCES_TO_DELETE.remove(instance) + def test_bigtable_test_iam_permissions(): # [START bigtable_test_iam_permissions] @@ -449,10 +492,6 @@ def test_bigtable_project_path(): project_path = client.project_path # [END bigtable_project_path] - _project_path = r"^projects/(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - _project_path_re = re.compile(_project_path) - assert _project_path_re.match(project_path) - def test_bigtable_table_data_client(): # [START bigtable_table_data_client] @@ -461,8 +500,6 @@ def test_bigtable_table_data_client(): client = Client(admin=True) table_data_client = client.table_data_client # [END bigtable_table_data_client] - assert "BigtableClient" in str(table_data_client) - def test_bigtable_table_admin_client(): # [START bigtable_table_admin_client] @@ -471,7 +508,6 @@ def test_bigtable_table_admin_client(): client = Client(admin=True) table_admin_client = client.table_admin_client # [END bigtable_table_admin_client] - assert "BigtableTableAdmin" in str(table_admin_client) def test_bigtable_instance_admin_client(): @@ -481,7 +517,6 @@ def test_bigtable_instance_admin_client(): client = Client(admin=True) instance_admin_client = client.instance_admin_client # [END bigtable_instance_admin_client] - assert "BigtableInstanceAdmin" in str(instance_admin_client) def test_bigtable_admins_policy(): @@ -579,13 +614,6 @@ def test_bigtable_instance_name(): instance_name = instance.name # [END bigtable_instance_name] - _instance_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P" - r"[a-z][-a-z0-9]*)$" - ) - assert _instance_name_re.match(instance_name) - def test_bigtable_cluster_name(): import re @@ -599,15 +627,6 @@ def test_bigtable_cluster_name(): cluster_name = cluster.name # [END bigtable_cluster_name] - _cluster_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/" - r"clusters/(?P" - r"[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - - assert _cluster_name_re.match(cluster_name) - def test_bigtable_instance_from_pb(): # [START bigtable_instance_from_pb] @@ -624,6 +643,7 @@ def test_bigtable_instance_from_pb(): instance2 = instance.from_pb(instance_pb, client) # [END bigtable_instance_from_pb] + assert instance2.name == instance.name @@ -648,6 +668,7 @@ def test_bigtable_cluster_from_pb(): cluster2 = cluster.from_pb(cluster_pb, instance) # [END bigtable_cluster_from_pb] + assert cluster2.name == cluster.name @@ -659,6 +680,7 @@ def test_bigtable_instance_state(): instance = client.instance(INSTANCE_ID) instance_state = instance.state # [END bigtable_instance_state] + assert not instance_state diff --git a/bigtable/tests/system.py b/bigtable/tests/system.py index 3631cf17e14a..7c00f56eaa24 100644 --- a/bigtable/tests/system.py +++ b/bigtable/tests/system.py @@ -39,9 +39,10 @@ from test_utils.system import EmulatorCreds from test_utils.system import unique_resource_id +UNIQUE_SUFFIX = unique_resource_id("-") LOCATION_ID = "us-central1-c" -INSTANCE_ID = "g-c-p" + unique_resource_id("-") -INSTANCE_ID_DATA = "g-c-p-d" + unique_resource_id("-") +INSTANCE_ID = "g-c-p" + UNIQUE_SUFFIX +INSTANCE_ID_DATA = "g-c-p-d" + UNIQUE_SUFFIX TABLE_ID = "google-cloud-python-test-table" CLUSTER_ID = INSTANCE_ID + "-cluster" CLUSTER_ID_DATA = INSTANCE_ID_DATA + "-cluster" @@ -125,10 +126,10 @@ def setUpModule(): EXISTING_INSTANCES[:] = instances # After listing, create the test instances. - created_op = Config.INSTANCE.create(clusters=[Config.CLUSTER]) - created_op.result(timeout=10) - created_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA]) - created_op.result(timeout=10) + admin_op = Config.INSTANCE.create(clusters=[Config.CLUSTER]) + admin_op.result(timeout=10) + data_op = Config.INSTANCE_DATA.create(clusters=[Config.CLUSTER_DATA]) + data_op.result(timeout=10) def tearDownModule(): @@ -140,7 +141,7 @@ def tearDownModule(): class TestInstanceAdminAPI(unittest.TestCase): def setUp(self): if Config.IN_EMULATOR: - self.skipTest("Instance Admin API not supported in Bigtable emulator") + self.skipTest("Instance Admin API not supported in emulator") self.instances_to_delete = [] def tearDown(self): @@ -172,7 +173,7 @@ def test_reload(self): def test_create_instance_defaults(self): from google.cloud.bigtable import enums - ALT_INSTANCE_ID = "ndef" + unique_resource_id("-") + ALT_INSTANCE_ID = "ndef" + UNIQUE_SUFFIX instance = Config.CLIENT.instance(ALT_INSTANCE_ID, labels=LABELS) ALT_CLUSTER_ID = ALT_INSTANCE_ID + "-cluster" cluster = instance.cluster( @@ -200,9 +201,8 @@ def test_create_instance(self): from google.cloud.bigtable import enums _DEVELOPMENT = enums.Instance.Type.DEVELOPMENT - _STATE = enums.Instance.State.READY - ALT_INSTANCE_ID = "new" + unique_resource_id("-") + ALT_INSTANCE_ID = "new" + UNIQUE_SUFFIX instance = Config.CLIENT.instance( ALT_INSTANCE_ID, instance_type=_DEVELOPMENT, labels=LABELS ) @@ -224,7 +224,7 @@ def test_create_instance(self): self.assertEqual(instance.display_name, instance_alt.display_name) self.assertEqual(instance.type_, instance_alt.type_) self.assertEqual(instance_alt.labels, LABELS) - self.assertEqual(_STATE, instance_alt.state) + self.assertEqual(instance_alt.state, enums.Instance.State.READY) def test_cluster_exists(self): NONEXISTING_CLUSTER_ID = "cluster-id" @@ -246,7 +246,7 @@ def test_create_instance_w_two_clusters(self): from google.cloud.bigtable.table import ClusterState _PRODUCTION = enums.Instance.Type.PRODUCTION - ALT_INSTANCE_ID = "dif" + unique_resource_id("-") + ALT_INSTANCE_ID = "dif" + UNIQUE_SUFFIX instance = Config.CLIENT.instance( ALT_INSTANCE_ID, instance_type=_PRODUCTION, labels=LABELS ) @@ -273,7 +273,7 @@ def test_create_instance_w_two_clusters(self): self.instances_to_delete.append(instance) # We want to make sure the operation completes. - operation.result(timeout=10) + operation.result(timeout=30) # Create a new instance instance and make sure it is the same. instance_alt = Config.CLIENT.instance(ALT_INSTANCE_ID) @@ -466,11 +466,12 @@ def test_update_type(self): _DEVELOPMENT = Instance.Type.DEVELOPMENT _PRODUCTION = Instance.Type.PRODUCTION - ALT_INSTANCE_ID = "ndif" + unique_resource_id("-") + ALT_INSTANCE_ID = "ndif" + UNIQUE_SUFFIX instance = Config.CLIENT.instance( ALT_INSTANCE_ID, instance_type=_DEVELOPMENT, labels=LABELS ) operation = instance.create(location_id=LOCATION_ID, serve_nodes=None) + # Make sure this instance gets deleted after the test case. self.instances_to_delete.append(instance) @@ -530,7 +531,7 @@ def test_create_cluster(self): operation = cluster_2.create() # We want to make sure the operation completes. - operation.result(timeout=10) + operation.result(timeout=30) # Create a new object instance, reload and make sure it is the same. alt_cluster = Config.INSTANCE.cluster(ALT_CLUSTER_ID)