diff --git a/.coveragerc b/.coveragerc index 04092257a..1ed1a9704 100644 --- a/.coveragerc +++ b/.coveragerc @@ -12,3 +12,8 @@ exclude_lines = pragma: NO COVER # Ignore debug-only repr def __repr__ + # Ignore pkg_resources exceptions. + # This is added at the module level as a safeguard for if someone + # generates the code and tries to run it without pip installing. This + # makes it virtually impossible to test properly. + except pkg_resources.DistributionNotFound diff --git a/google/__init__.py b/google/__init__.py new file mode 100644 index 000000000..8e60d8439 --- /dev/null +++ b/google/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import pkg_resources + + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/google/cloud/__init__.py b/google/cloud/__init__.py new file mode 100644 index 000000000..8e60d8439 --- /dev/null +++ b/google/cloud/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import pkg_resources + + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/noxfile.py b/noxfile.py index 246952728..9ccbdd30c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -137,7 +137,7 @@ def mypy(session): "types-requests", "types-setuptools", ) - session.run("mypy", "-p", "google", "--show-traceback") + session.run("mypy", "google/cloud", "--show-traceback") @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -149,8 +149,7 @@ def pytype(session): session.install("attrs==20.3.0") session.install("-e", ".[all]") session.install(PYTYPE_VERSION) - # See https://github.com/google/pytype/issues/464 - session.run("pytype", "-P", ".", "google/cloud/bigquery") + session.run("pytype") @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) diff --git a/setup.py b/setup.py index f21bb586d..ead602e12 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ "pandas>=1.1.0", pyarrow_dependency, "db-dtypes>=0.3.0,<2.0.0dev", - "importlib_metadata>=1.0.0; python_version<'3.8'", ], "ipywidgets": [ "ipywidgets>=7.7.0", @@ -109,10 +108,16 @@ # benchmarks, etc. packages = [ package - for package in setuptools.find_namespace_packages() + for package in setuptools.PEP420PackageFinder.find() if package.startswith("google") ] +# Determine which namespaces are needed. +namespaces = ["google"] +if "google.cloud" in packages: + namespaces.append("google.cloud") + + setuptools.setup( name=name, version=version, @@ -138,6 +143,7 @@ ], platforms="Posix; MacOS X; Windows", packages=packages, + namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, python_requires=">=3.7", diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index e93f245c0..9f7fc242e 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -23,13 +23,9 @@ import warnings import google.api_core.retry +import pkg_resources import pytest -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata - from google.cloud import bigquery from google.cloud.bigquery import enums @@ -46,9 +42,11 @@ ) if pandas is not None: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: - PANDAS_INSTALLED_VERSION = "0.0.0" + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + +PANDAS_INT64_VERSION = pkg_resources.parse_version("1.0.0") class MissingDataError(Exception): @@ -312,7 +310,10 @@ def test_load_table_from_dataframe_w_automatic_schema(bigquery_client, dataset_i ] -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.skipif( + PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION, + reason="Only `pandas version >=1.0.0` is supported", +) def test_load_table_from_dataframe_w_nullable_int64_datatype( bigquery_client, dataset_id ): @@ -341,7 +342,7 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype( @pytest.mark.skipif( - PANDAS_INSTALLED_VERSION[0:2].startswith("0."), + PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION, reason="Only `pandas version >=1.0.0` is supported", ) def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema( @@ -1042,7 +1043,9 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client): assert len(dataframe.index) == 100 -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") +@pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" +) @pytest.mark.parametrize( ("max_results",), ( diff --git a/tests/unit/job/test_query_pandas.py b/tests/unit/job/test_query_pandas.py index 6189830ff..0accae0a2 100644 --- a/tests/unit/job/test_query_pandas.py +++ b/tests/unit/job/test_query_pandas.py @@ -17,6 +17,7 @@ import json import mock +import pkg_resources import pytest @@ -44,19 +45,14 @@ except (ImportError, AttributeError): # pragma: NO COVER tqdm = None -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata - from ..helpers import make_connection from .helpers import _make_client from .helpers import _make_job_resource if pandas is not None: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: - PANDAS_INSTALLED_VERSION = "0.0.0" + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") pandas = pytest.importorskip("pandas") @@ -660,7 +656,9 @@ def test_to_dataframe_bqstorage_no_pyarrow_compression(): ) -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") +@pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" +) @pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") def test_to_dataframe_column_dtypes(): from google.cloud.bigquery.job import QueryJob as target_class diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index ad40a6da6..1f1b4eeb3 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -19,11 +19,7 @@ import operator import queue import warnings - -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata +import pkg_resources import mock @@ -61,10 +57,13 @@ bigquery_storage = _versions_helpers.BQ_STORAGE_VERSIONS.try_import() +PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0") + if pandas is not None: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: - PANDAS_INSTALLED_VERSION = "0.0.0" + # Set to less than MIN version. + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") skip_if_no_bignumeric = pytest.mark.skipif( @@ -543,7 +542,9 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows): ], ) @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") +@pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" +) @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows): rows = [pandas.Timestamp(row) for row in rows] @@ -805,7 +806,10 @@ def test_list_columns_and_indexes_with_named_index_same_as_column_name( assert columns_and_indexes == expected -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.skipif( + pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, + reason="Requires `pandas version >= 1.0.0` which introduces pandas.NA", +) def test_dataframe_to_json_generator(module_under_test): utcnow = datetime.datetime.utcnow() df_data = collections.OrderedDict( @@ -833,8 +837,16 @@ def test_dataframe_to_json_generator(module_under_test): assert list(rows) == expected -@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_dataframe_to_json_generator_repeated_field(module_under_test): + pytest.importorskip( + "pandas", + minversion=str(PANDAS_MINIUM_VERSION), + reason=( + f"Requires `pandas version >= {PANDAS_MINIUM_VERSION}` " + "which introduces pandas.NA" + ), + ) + df_data = [ collections.OrderedDict( [("repeated_col", [pandas.NA, 2, None, 4]), ("not_repeated_col", "first")] diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index ff4c40f48..af61ceb42 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -30,11 +30,7 @@ import requests import packaging import pytest - -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata +import pkg_resources try: import pandas @@ -80,10 +76,13 @@ from test_utils.imports import maybe_fail_import from tests.unit.helpers import make_connection +PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0") + if pandas is not None: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: - PANDAS_INSTALLED_VERSION = "0.0.0" + # Set to less than MIN version. + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") def _make_credentials(): @@ -8146,7 +8145,10 @@ def test_load_table_from_dataframe_unknown_table(self): timeout=DEFAULT_TIMEOUT, ) - @unittest.skipIf(pandas is None, "Requires `pandas`") + @unittest.skipIf( + pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, + "Only `pandas version >=1.0.0` supported", + ) @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_load_table_from_dataframe_w_nullable_int64_datatype(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES @@ -8191,7 +8193,10 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype(self): SchemaField("x", "INT64", "NULLABLE", None), ) - @unittest.skipIf(pandas is None, "Requires `pandas`") + @unittest.skipIf( + pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, + "Only `pandas version >=1.0.0` supported", + ) # @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py deleted file mode 100644 index 6f1b16c66..000000000 --- a/tests/unit/test_packaging.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import sys - - -def test_namespace_package_compat(tmp_path): - # The ``google`` namespace package should not be masked - # by the presence of ``google-cloud-bigquery``. - google = tmp_path / "google" - google.mkdir() - google.joinpath("othermod.py").write_text("") - env = dict(os.environ, PYTHONPATH=str(tmp_path)) - cmd = [sys.executable, "-m", "google.othermod"] - subprocess.check_call(cmd, env=env) - - # The ``google.cloud`` namespace package should not be masked - # by the presence of ``google-cloud-bigquery``. - google_cloud = tmp_path / "google" / "cloud" - google_cloud.mkdir() - google_cloud.joinpath("othermod.py").write_text("") - env = dict(os.environ, PYTHONPATH=str(tmp_path)) - cmd = [sys.executable, "-m", "google.cloud.othermod"] - subprocess.check_call(cmd, env=env) diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 85f335dd1..05ad8de6e 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -22,13 +22,9 @@ import warnings import mock +import pkg_resources import pytest -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata - import google.api_core.exceptions from test_utils.imports import maybe_fail_import @@ -75,9 +71,9 @@ tqdm = None if pandas is not None: - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: - PANDAS_INSTALLED_VERSION = "0.0.0" + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") def _mock_client(): @@ -3797,7 +3793,9 @@ def test_to_dataframe_w_dtypes_mapper(self): self.assertEqual(df.timestamp.dtype.name, "object") @unittest.skipIf(pandas is None, "Requires `pandas`") - @pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") + @pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" + ) def test_to_dataframe_w_none_dtypes_mapper(self): from google.cloud.bigquery.schema import SchemaField @@ -3910,7 +3908,9 @@ def test_to_dataframe_w_unsupported_dtypes_mapper(self): ) @unittest.skipIf(pandas is None, "Requires `pandas`") - @pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") + @pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" + ) def test_to_dataframe_column_dtypes(self): from google.cloud.bigquery.schema import SchemaField diff --git a/tests/unit/test_table_pandas.py b/tests/unit/test_table_pandas.py index b38568561..6970d9d65 100644 --- a/tests/unit/test_table_pandas.py +++ b/tests/unit/test_table_pandas.py @@ -15,11 +15,7 @@ import datetime import decimal from unittest import mock - -try: - import importlib.metadata as metadata -except ImportError: - import importlib_metadata as metadata +import pkg_resources import pytest @@ -32,9 +28,9 @@ TEST_PATH = "/v1/project/test-proj/dataset/test-dset/table/test-tbl/data" if pandas is not None: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = metadata.version("pandas") + PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version else: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = "0.0.0" + PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") @pytest.fixture @@ -44,7 +40,9 @@ def class_under_test(): return RowIterator -@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") +@pytest.mark.skipif( + PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" +) def test_to_dataframe_nullable_scalars(monkeypatch, class_under_test): # See tests/system/test_arrow.py for the actual types we get from the API. arrow_schema = pyarrow.schema(