tests: allow prerelease dependency versions under Python 3.9 (#479)

Also, drop use of 'pytz', which is no longer depended on by `google-api-core` / `google-cloud-core`.

Instead, use either `datetime.timezone.utc` or `google.cloud._helpers.UTC`, depending on usage.

Author: tseaver

docs/snapshot-usage.rst

@@ -24,8 +24,7 @@ reads as of a given timestamp:
 .. code:: python
 
     import datetime
-    from pytz import UTC
-    TIMESTAMP = datetime.datetime.utcnow().replace(tzinfo=UTC)
+    TIMESTAMP = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
 
     with database.snapshot(read_timestamp=TIMESTAMP) as snapshot:
         ...

noxfile.py

@@ -105,7 +105,11 @@ def default(session):
         *session.posargs,
     )
 
+    # XXX Work around Kokoro image's older pip, which borks the OT install.
+    session.run("pip", "install", "--upgrade", "pip")
     session.install("-e", ".[tracing]", "-c", constraints_path)
+    # XXX: Dump installed versions to debug OT issue
+    session.run("pip", "list")
 
     # Run py.test against the unit tests with OpenTelemetry.
     session.run(

owlbot.py

@@ -23,12 +23,16 @@
 
 common = gcp.CommonTemplates()
 
-# This is a customized version of the s.get_staging_dirs() function from synthtool to
-# cater for copying 3 different folders from googleapis-gen
-# which are spanner, spanner/admin/instance and spanner/admin/database.
-# Source https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280
+
 def get_staging_dirs(
-    default_version: Optional[str] = None, sub_directory: Optional[str] = None
+    # This is a customized version of the s.get_staging_dirs() function
+    # from synthtool to # cater for copying 3 different folders from
+    # googleapis-gen:
+    # spanner, spanner/admin/instance and spanner/admin/database.
+    # Source:
+    # https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280
+    default_version: Optional[str] = None,
+    sub_directory: Optional[str] = None,
 ) -> List[Path]:
     """Returns the list of directories, one per version, copied from
     https://github.com/googleapis/googleapis-gen. Will return in lexical sorting
@@ -63,46 +67,69 @@ def get_staging_dirs(
     else:
         return []
 
+
 spanner_default_version = "v1"
 spanner_admin_instance_default_version = "v1"
 spanner_admin_database_default_version = "v1"
 
 for library in get_staging_dirs(spanner_default_version, "spanner"):
     # Work around gapic generator bug https://github.com/googleapis/gapic-generator-python/issues/902
-    s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
-            r""".
+    s.replace(
+        library / f"google/cloud/spanner_{library.name}/types/transaction.py",
+        r""".
         Attributes:""",
-            r""".\n
+        r""".\n
         Attributes:""",
     )
 
     # Work around gapic generator bug https://github.com/googleapis/gapic-generator-python/issues/902
-    s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
-            r""".
+    s.replace(
+        library / f"google/cloud/spanner_{library.name}/types/transaction.py",
+        r""".
     Attributes:""",
-            r""".\n
+        r""".\n
     Attributes:""",
     )
 
     # Remove headings from docstring. Requested change upstream in cl/377290854 due to https://google.aip.dev/192#formatting.
-    s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
+    s.replace(
+        library / f"google/cloud/spanner_{library.name}/types/transaction.py",
         """\n    ==.*?==\n""",
         ":",
     )
 
     # Remove headings from docstring. Requested change upstream in cl/377290854 due to https://google.aip.dev/192#formatting.
-    s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
+    s.replace(
+        library / f"google/cloud/spanner_{library.name}/types/transaction.py",
         """\n    --.*?--\n""",
         ":",
     )
 
-    s.move(library, excludes=["google/cloud/spanner/**", "*.*", "docs/index.rst", "google/cloud/spanner_v1/__init__.py"])
+    s.move(
+        library,
+        excludes=[
+            "google/cloud/spanner/**",
+            "*.*",
+            "docs/index.rst",
+            "google/cloud/spanner_v1/__init__.py",
+        ],
+    )
 
-for library in get_staging_dirs(spanner_admin_instance_default_version, "spanner_admin_instance"):
-    s.move(library, excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst"])
+for library in get_staging_dirs(
+    spanner_admin_instance_default_version, "spanner_admin_instance"
+):
+    s.move(
+        library,
+        excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst"],
+    )
 
-for library in get_staging_dirs(spanner_admin_database_default_version, "spanner_admin_database"):
-    s.move(library, excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst"])
+for library in get_staging_dirs(
+    spanner_admin_database_default_version, "spanner_admin_database"
+):
+    s.move(
+        library,
+        excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst"],
+    )
 
 s.remove_staging_dirs()
 
@@ -116,9 +143,11 @@ def get_staging_dirs(
 s.replace(
     ".kokoro/build.sh",
     "# Remove old nox",
-    "# Set up creating a new instance for each system test run\n"
-    "export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true\n"
-    "\n\g<0>",
+    """\
+# Set up creating a new instance for each system test run
+export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true
+
+# Remove old nox""",
 )
 
 # Update samples folder in CONTRIBUTING.rst
@@ -134,15 +163,21 @@ def get_staging_dirs(
 # Customize noxfile.py
 # ----------------------------------------------------------------------------
 
+
 def place_before(path, text, *before_text, escape=None):
     replacement = "\n".join(before_text) + "\n" + text
     if escape:
         for c in escape:
-            text = text.replace(c, '\\' + c)
+            text = text.replace(c, "\\" + c)
     s.replace([path], text, replacement)
 
+
 open_telemetry_test = """
+    # XXX Work around Kokoro image's older pip, which borks the OT install.
+    session.run("pip", "install", "--upgrade", "pip")
     session.install("-e", ".[tracing]", "-c", constraints_path)
+    # XXX: Dump installed versions to debug OT issue
+    session.run("pip", "list")
 
     # Run py.test against the unit tests with OpenTelemetry.
     session.run(
@@ -164,10 +199,10 @@ def place_before(path, text, *before_text, escape=None):
     "noxfile.py",
     "@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)",
     open_telemetry_test,
-    escape="()"
+    escape="()",
 )
 
-skip_tests_if_env_var_not_set ="""# Sanity check: Only run tests if the environment variable is set.
+skip_tests_if_env_var_not_set = """# Sanity check: Only run tests if the environment variable is set.
     if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") and not os.environ.get(
         "SPANNER_EMULATOR_HOST", ""
     ):
@@ -180,7 +215,7 @@ def place_before(path, text, *before_text, escape=None):
     "noxfile.py",
     "# Install pyopenssl for mTLS testing.",
     skip_tests_if_env_var_not_set,
-    escape="()"
+    escape="()",
 )
 
 s.replace(
@@ -190,25 +225,25 @@ def place_before(path, text, *before_text, escape=None):
         "--cov=tests/unit",""",
     """\"--cov=google.cloud.spanner",
         "--cov=google.cloud",
-        "--cov=tests.unit","""
+        "--cov=tests.unit",""",
 )
 
 s.replace(
     "noxfile.py",
-    """session.install\("-e", "."\)""",
-    """session.install("-e", ".[tracing]")"""
+    r"""session.install\("-e", "."\)""",
+    """session.install("-e", ".[tracing]")""",
 )
 
 s.replace(
     "noxfile.py",
-    """# Install all test dependencies, then install this package into the
+    r"""# Install all test dependencies, then install this package into the
     # virtualenv's dist-packages.
     session.install\("mock", "pytest", "google-cloud-testutils", "-c", constraints_path\)
     session.install\("-e", ".", "-c", constraints_path\)""",
     """# Install all test dependencies, then install this package into the
     # virtualenv's dist-packages.
     session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path)
-    session.install("-e", ".[tracing]", "-c", constraints_path)"""
+    session.install("-e", ".[tracing]", "-c", constraints_path)""",
 )
 
 s.shell.run(["nox", "-s", "blacken"], hide_output=False)

samples/samples/conftest.py

@@ -24,6 +24,8 @@
 import pytest
 from test_utils import retry
 
+retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
+
 
 @pytest.fixture(scope="module")
 def sample_name():
@@ -47,7 +49,7 @@ def scrub_instance_ignore_not_found(to_scrub):
         for backup_pb in to_scrub.list_backups():
             backup.Backup.from_pb(backup_pb, to_scrub).delete()
 
-        to_scrub.delete()
+        retry_429(to_scrub.delete)()
     except exceptions.NotFound:
         pass
 
@@ -107,7 +109,6 @@ def sample_instance(
             "created": str(int(time.time())),
         },
     )
-    retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
     op = retry_429(sample_instance.create)()
     op.result(120)  # block until completion
 
@@ -143,7 +144,6 @@ def multi_region_instance(
             "created": str(int(time.time()))
         },
     )
-    retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
     op = retry_429(multi_region_instance.create)()
     op.result(120)  # block until completion
 

samples/samples/snippets.py

@@ -30,6 +30,8 @@
 from google.cloud import spanner
 from google.cloud.spanner_v1 import param_types
 
+OPERATION_TIMEOUT_SECONDS = 240
+
 
 # [START spanner_create_instance]
 def create_instance(instance_id):
@@ -55,7 +57,7 @@ def create_instance(instance_id):
     operation = instance.create()
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Created instance {}".format(instance_id))
 
@@ -87,7 +89,7 @@ def create_instance_with_processing_units(instance_id, processing_units):
     operation = instance.create()
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Created instance {} with {} processing units".format(
         instance_id, instance.processing_units))
@@ -170,7 +172,7 @@ def create_database(instance_id, database_id):
     operation = database.create()
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Created database {} on instance {}".format(database_id, instance_id))
 
@@ -206,7 +208,7 @@ def create_database_with_encryption_key(instance_id, database_id, kms_key_name):
     operation = database.create()
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Database {} created with encryption key {}".format(
         database.name, database.encryption_config.kms_key_name))
@@ -245,7 +247,7 @@ def create_database_with_default_leader(
     operation = database.create()
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     database.reload()
 
@@ -271,7 +273,7 @@ def update_database_with_default_leader(
 
     operation = database.update_ddl(["ALTER DATABASE {}"
                                      " SET OPTIONS (default_leader = '{}')".format(database_id, default_leader)])
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     database.reload()
 
@@ -499,7 +501,7 @@ def add_index(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Added the AlbumsByAlbumTitle index.")
 
@@ -598,7 +600,7 @@ def add_storing_index(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Added the AlbumsByAlbumTitle2 index.")
 
@@ -651,7 +653,7 @@ def add_column(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print("Added the MarketingBudget column.")
 
@@ -816,7 +818,7 @@ def create_table_with_timestamp(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print(
         "Created Performances table on database {} on instance {}".format(
@@ -871,7 +873,7 @@ def add_timestamp_column(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print(
         'Altered table "Albums" on database {} on instance {}.'.format(
@@ -964,7 +966,7 @@ def add_numeric_column(instance_id, database_id):
     operation = database.update_ddl(["ALTER TABLE Venues ADD COLUMN Revenue NUMERIC"])
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print(
         'Altered table "Venues" on database {} on instance {}.'.format(
@@ -1564,7 +1566,7 @@ def create_table_with_datatypes(instance_id, database_id):
     )
 
     print("Waiting for operation to complete...")
-    operation.result(120)
+    operation.result(OPERATION_TIMEOUT_SECONDS)
 
     print(
         "Created Venues table on database {} on instance {}".format(