From 6874109c7335137d669bce97f36ccc6cfca8bbb6 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Oct 2024 11:52:21 +0200 Subject: [PATCH 01/10] Use pyproject.toml --- doc/contributing.rst | 4 +- pyproject.toml | 145 ++++++++++++++++++++++++++ setup.cfg | 32 ------ setup.py | 239 ------------------------------------------- 4 files changed, 147 insertions(+), 273 deletions(-) delete mode 100755 setup.py diff --git a/doc/contributing.rst b/doc/contributing.rst index a21a005e72..d432c6e4e5 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -443,7 +443,7 @@ previous command. To only run tests from a single file, run the command pytest tests/unit/test_some_file.py If you would like to avoid loading the default pytest configuration from -`setup.cfg `_ +`pyproject.toml `_ because this can be a bit slow for running just a few tests, use .. code-block:: bash @@ -653,7 +653,7 @@ the following files: - ``environment.yml`` contains all the development dependencies; these are all from `conda-forge `_ -- ``setup.py`` +- ``pyproject.toml`` contains all Python dependencies, regardless of their installation source Note that packages may have a different name on diff --git a/pyproject.toml b/pyproject.toml index 5a45ca2ab9..6c8854f40c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,117 @@ requires = ["setuptools >= 40.6.0", "wheel", "setuptools_scm>=6.2"] build-backend = "setuptools.build_meta" +[project] +authors = [ + {name = "ESMValTool Development Team", email = "esmvaltool-dev@listserv.dfn.de"} +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Physics", +] +dynamic = [ + "readme", + "version", +] +description = "A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts" +dependencies = [ + "cartopy", + "cf-units", + "dask[array,distributed]!=2024.8.0", # ESMValCore/issues/2503 + "dask-jobqueue", + "esgf-pyclient>=0.3.1", + "esmf-regrid>=0.11.0", + "esmpy!=8.1.0", # not on PyPI + "filelock", + "fiona", + "fire", + "geopy", + "humanfriendly", + "iris-grib", + "isodate", + "jinja2", + "nc-time-axis", # needed by iris.plot + "nested-lookup", + "netCDF4", + "numpy!=1.24.3,<2.0.0", # avoid pulling 2.0.0rc1 + "packaging", + "pandas", + "pillow", + "prov", + "psutil", + "py-cordex", + "pybtex", + "pyyaml", + "requests", + "scipy>=1.6", + "scitools-iris>=3.10.0", + "shapely>=2.0.0", + "stratify>=0.3", + "yamale", +] +license = {text = "Apache License, Version 2.0"} +name = "ESMValCore" +requires-python = ">=3.10" + +[project.optional-dependencies] +test = [ + "pytest>=3.9,!=6.0.0rc1,!=6.0.0", + "pytest-cov>=2.10.1", + "pytest-env", + "pytest-html!=2.1.0", + "pytest-metadata>=1.5.1", + "pytest-mock", + "pytest-xdist", + "ESMValTool_sample_data==0.0.3", +] +doc = [ + "autodocsumm>=0.2.2", + "ipython", + "nbsphinx", + "sphinx>=6.1.3", + "pydata_sphinx_theme", +] +develop = [ + "esmvalcore[test,doc]", + "pre-commit", + "pylint", + "pydocstyle", + "vprof", +] + +[project.scripts] +esmvaltool = "esmvalcore._main:run" + +[project.urls] +Code = "https://github.com/ESMValGroup/ESMValCore" +Community = "https://github.com/ESMValGroup/Community" +Documentation = "https://docs.esmvaltool.org" +Homepage = "https://esmvaltool.org" +Issues = "https://github.com/ESMValGroup/ESMValCore/issues" + +[tool.setuptools] +include-package-data = true +license-files = ["LICENSE"] +packages = ["esmvalcore"] +zip-safe = false + +[tool.setuptools.dynamic] +readme = {file = "README.md", content-type = "text/markdown"} + [tool.setuptools_scm] version_scheme = "release-branch-semver" @@ -29,6 +140,33 @@ disable = [ "line-too-long", # Disable line-too-long as this is taken care of by the formatter. "locally-disabled", # Disable messages about disabling checks ] + +[tool.pytest.ini_options] +addopts = """\ + --doctest-modules \ + --ignore=esmvalcore/cmor/tables/ \ + --cov=esmvalcore \ + --cov-report=xml:test-reports/coverage.xml \ + --cov-report=html:test-reports/coverage_html \ + --html=test-reports/report.html \ +""" +env = {MPLBACKEND = "Agg"} +log_level = "WARNING" +markers = [ + "installation: Test requires installation of dependencies", + "use_sample_data: Run functional tests using real data", +] + +[tool.coverage.run] +parallel = true + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + [tool.ruff] line-length = 79 [tool.ruff.lint] @@ -44,3 +182,10 @@ ignore = [ ] [tool.ruff.lint.isort] known-first-party = ["esmvalcore"] + +[tool.mypy] +# See https://mypy.readthedocs.io/en/stable/config_file.html +ignore_missing_imports = true + +[tool.pydocstyle] +convention = "numpy" diff --git a/setup.cfg b/setup.cfg index 15d02392d7..3597a6ae5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,38 +1,6 @@ -[tool:pytest] -addopts = - --doctest-modules - --ignore=esmvalcore/cmor/tables/ - --cov=esmvalcore - --cov-report=xml:test-reports/coverage.xml - --cov-report=html:test-reports/coverage_html - --html=test-reports/report.html -env = - MPLBACKEND = Agg -log_level = WARNING -markers = - installation: Test requires installation of dependencies - use_sample_data: Run functional tests using real data - -[coverage:run] -parallel = true -[coverage:report] -exclude_lines = - pragma: no cover - if __name__ == .__main__.: - if TYPE_CHECKING: - [pycodestyle] # ignore rules that conflict with ruff formatter # E203: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices # E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules # W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes ignore = E203,E501,W503 - -[pydocstyle] -convention = numpy - -[mypy] -# see mypy.readthedocs.io/en/stable/command_line.html -python_version = 3.12 -ignore_missing_imports = True -files = esmvalcore, tests diff --git a/setup.py b/setup.py deleted file mode 100755 index 32d2e15e27..0000000000 --- a/setup.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python -"""ESMValTool installation script.""" -# This script only installs dependencies available on PyPI -# -# Dependencies that need to be installed some other way (e.g. conda): -# - ncl -# - iris -# - python-stratify - -import json -import os -import re -import sys -from pathlib import Path - -from setuptools import Command, setup - -PACKAGES = [ - "esmvalcore", -] - -REQUIREMENTS = { - # Installation script (this file) dependencies - "setup": [ - "setuptools_scm", - ], - # Installation dependencies - # Use with pip install . to install from source - "install": [ - "cartopy", - "cf-units", - "dask[array,distributed]!=2024.8.0", # ESMValCore/issues/2503 - "dask-jobqueue", - "esgf-pyclient>=0.3.1", - "esmf-regrid>=0.11.0", - "esmpy!=8.1.0", # not on PyPI - "filelock", - "fiona", - "fire", - "geopy", - "humanfriendly", - "iris-grib", - "isodate", - "jinja2", - "nc-time-axis", # needed by iris.plot - "nested-lookup", - "netCDF4", - "numpy!=1.24.3,<2.0.0", # avoid pulling 2.0.0rc1 - "packaging", - "pandas", - "pillow", - "prov", - "psutil", - "py-cordex", - "pybtex", - "pyyaml", - "requests", - "scipy>=1.6", - "scitools-iris>=3.10.0", - "shapely>=2.0.0", - "stratify>=0.3", - "yamale", - ], - # Test dependencies - "test": [ - "pytest>=3.9,!=6.0.0rc1,!=6.0.0", - "pytest-cov>=2.10.1", - "pytest-env", - "pytest-html!=2.1.0", - "pytest-metadata>=1.5.1", - "pytest-mock", - "pytest-xdist", - "ESMValTool_sample_data==0.0.3", - ], - # Documentation dependencies - "doc": [ - "autodocsumm>=0.2.2", - "ipython", - "nbsphinx", - "sphinx>=6.1.3", - "pydata_sphinx_theme", - ], - # Development dependencies - # Use pip install -e .[develop] to install in development mode - "develop": [ - "pre-commit", - "pylint", - "pydocstyle", - "vprof", - ], -} - - -def discover_python_files(paths, ignore): - """Discover Python files.""" - - def _ignore(path): - """Return True if `path` should be ignored, False otherwise.""" - return any(re.match(pattern, path) for pattern in ignore) - - for path in sorted(set(paths)): - for root, _, files in os.walk(path): - if _ignore(path): - continue - for filename in files: - filename = os.path.join(root, filename) - if filename.lower().endswith(".py") and not _ignore(filename): - yield filename - - -class CustomCommand(Command): - """Custom Command class.""" - - def install_deps_temp(self): - """Try to temporarily install packages needed to run the command.""" - if self.distribution.install_requires: - self.distribution.fetch_build_eggs( - self.distribution.install_requires - ) - if self.distribution.tests_require: - self.distribution.fetch_build_eggs(self.distribution.tests_require) - - -class RunLinter(CustomCommand): - """Class to run a linter and generate reports.""" - - user_options: list = [] - - def initialize_options(self): - """Do nothing.""" - - def finalize_options(self): - """Do nothing.""" - - def run(self): - """Run prospector and generate a report.""" - check_paths = PACKAGES + [ - "setup.py", - "tests", - ] - ignore = [ - "doc/", - ] - - # try to install missing dependencies and import prospector - try: - from prospector.run import main - except ImportError: - # try to install and then import - self.distribution.fetch_build_eggs(["prospector[with_pyroma]"]) - from prospector.run import main - - self.install_deps_temp() - - # run linter - - # change working directory to package root - package_root = os.path.abspath(os.path.dirname(__file__)) - os.chdir(package_root) - - # write command line - files = discover_python_files(check_paths, ignore) - sys.argv = ["prospector"] - sys.argv.extend(files) - - # run prospector - errno = main() - - sys.exit(errno) - - -def read_authors(filename): - """Read the list of authors from .zenodo.json file.""" - with Path(filename).open(encoding="utf-8") as file: - info = json.load(file) - authors = [] - for author in info["creators"]: - name = " ".join(author["name"].split(",")[::-1]).strip() - authors.append(name) - return ", ".join(authors) - - -def read_description(filename): - """Read the description from .zenodo.json file.""" - with Path(filename).open(encoding="utf-8") as file: - info = json.load(file) - return info["description"] - - -setup( - name="ESMValCore", - author=read_authors(".zenodo.json"), - description=read_description(".zenodo.json"), - long_description=Path("README.md").read_text(encoding="utf-8"), - long_description_content_type="text/markdown", - url="https://www.esmvaltool.org", - download_url="https://github.com/ESMValGroup/ESMValCore", - license="Apache License, Version 2.0", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Atmospheric Science", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Hydrology", - "Topic :: Scientific/Engineering :: Physics", - ], - packages=PACKAGES, - # Include all version controlled files - include_package_data=True, - setup_requires=REQUIREMENTS["setup"], - install_requires=REQUIREMENTS["install"], - tests_require=REQUIREMENTS["test"], - extras_require={ - "develop": REQUIREMENTS["develop"] - + REQUIREMENTS["test"] - + REQUIREMENTS["doc"], - "test": REQUIREMENTS["test"], - "doc": REQUIREMENTS["doc"], - }, - entry_points={ - "console_scripts": [ - "esmvaltool = esmvalcore._main:run", - ], - }, - cmdclass={ - "lint": RunLinter, - }, - zip_safe=False, -) From 7eda4275424998409003097c476ed90c9dcbdf2a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Oct 2024 11:59:21 +0200 Subject: [PATCH 02/10] Fix formatting issue --- esmvalcore/_task.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index dc15718169..cbb53e2691 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -609,9 +609,10 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log), open( - self.log, "ab" - ) as log: + with ( + resource_usage_logger(process.pid, self.resource_log), + open(self.log, "ab") as log, + ): last_line = [""] while returncode is None: returncode = process.poll() From ed28019d1563f7efc93f9b29e877fd364c8fba1a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Oct 2024 13:03:24 +0200 Subject: [PATCH 03/10] Use recommendations --- pyproject.toml | 31 +++++++++++++------ .../cmor/_fixes/cmip6/test_cesm2.py | 1 - .../cmor/_fixes/cmip6/test_gfdl_cm4.py | 1 - .../_multimodel/test_multimodel.py | 1 - 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6c8854f40c..c98c57fcf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,8 @@ [build-system] -requires = ["setuptools >= 40.6.0", "wheel", "setuptools_scm>=6.2"] +requires = [ + "setuptools >= 40.6.0", + "setuptools_scm>=6.2", +] build-backend = "setuptools.build_meta" [project] @@ -70,7 +73,7 @@ requires-python = ">=3.10" [project.optional-dependencies] test = [ - "pytest>=3.9,!=6.0.0rc1,!=6.0.0", + "pytest>6.0.0", "pytest-cov>=2.10.1", "pytest-env", "pytest-html!=2.1.0", @@ -142,23 +145,31 @@ disable = [ ] [tool.pytest.ini_options] -addopts = """\ - --doctest-modules \ - --ignore=esmvalcore/cmor/tables/ \ - --cov=esmvalcore \ - --cov-report=xml:test-reports/coverage.xml \ - --cov-report=html:test-reports/coverage_html \ - --html=test-reports/report.html \ -""" +addopts = [ + "-ra", + "--strict-config", + "--strict-markers", + "--doctest-modules", + "--ignore=esmvalcore/cmor/tables/", + "--cov", + "--cov-report=xml:test-reports/coverage.xml", + "--cov-report=html:test-reports/coverage_html", + "--html=test-reports/report.html", +] +log_cli_level = "INFO" env = {MPLBACKEND = "Agg"} log_level = "WARNING" +minversion = "6" markers = [ "installation: Test requires installation of dependencies", "use_sample_data: Run functional tests using real data", ] +testpaths = ["tests"] +xfail_strict = true [tool.coverage.run] parallel = true +source = ["esmvalcore"] [tool.coverage.report] exclude_lines = [ diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 1624a688ea..c123d7f856 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -61,7 +61,6 @@ def test_get_cl_fix(): ) -@pytest.mark.sequential @pytest.mark.skipif( sys.version_info < (3, 7, 6), reason="requires python3.7.6 or newer" ) diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py index 7e5fb81110..5f845d6d3d 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py @@ -49,7 +49,6 @@ def test_get_cl_fix(): ) -@pytest.mark.sequential def test_cl_fix_metadata(test_data_path): """Test ``fix_metadata`` for ``cl``.""" nc_path = test_data_path / "gfdl_cm4_cl.nc" diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 5bc5513451..0b60b0f191 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -701,7 +701,6 @@ def generate_cubes_with_non_overlapping_timecoords(): ) -@pytest.mark.xfail(reason="Multimodel statistics returns the original cubes.") def test_edge_case_time_no_overlap_fail(): """Test case when time coords do not overlap using span='overlap'. From 8e922f62912fa3cc50ce107941115b28d9ba3cc0 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Oct 2024 13:09:00 +0200 Subject: [PATCH 04/10] Enable extra check --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c98c57fcf1..7bf4e1748d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,6 +197,9 @@ known-first-party = ["esmvalcore"] [tool.mypy] # See https://mypy.readthedocs.io/en/stable/config_file.html ignore_missing_imports = true +enable_error_code = [ + "truthy-bool", +] [tool.pydocstyle] convention = "numpy" From 8e9204af9b4629fd1f0c10dab0324dfd561aaa47 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 10 Oct 2024 21:50:44 +0200 Subject: [PATCH 05/10] Remove setup.py and setup.cfg from list of installation files --- .circleci/install_triggers | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/install_triggers b/.circleci/install_triggers index 6945bf8979..c1c5a829d4 100644 --- a/.circleci/install_triggers +++ b/.circleci/install_triggers @@ -1,5 +1,3 @@ ^\.circleci/ ^environment\.yml$ ^pyproject.toml$ -^setup\.py$ -^setup\.cfg$ From 1a7c867f9cfb5a61175e007202c84464513d2557 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 15 Oct 2024 20:05:05 +0200 Subject: [PATCH 06/10] Remove checks for unsupported Python versions --- esmvalcore/_main.py | 6 +----- tests/integration/cmor/_fixes/cmip6/test_cesm2.py | 4 ---- tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py | 4 ---- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 451f228ae8..6978d50f0c 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -33,14 +33,10 @@ import logging import os import sys +from importlib.metadata import entry_points from pathlib import Path from typing import Optional -if (sys.version_info.major, sys.version_info.minor) < (3, 10): - from importlib_metadata import entry_points -else: - from importlib.metadata import entry_points # type: ignore - import fire # set up logging diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index c123d7f856..5d504f6084 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -1,7 +1,6 @@ """Tests for the fixes of CESM2.""" import os -import sys import unittest.mock import iris @@ -61,9 +60,6 @@ def test_get_cl_fix(): ) -@pytest.mark.skipif( - sys.version_info < (3, 7, 6), reason="requires python3.7.6 or newer" -) @unittest.mock.patch( "esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath", autospec=True ) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index dd400c8ab1..363bf0d80c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -1,7 +1,6 @@ """Tests for the fixes of CESM2-WACCM.""" import os -import sys import unittest.mock import iris @@ -36,9 +35,6 @@ def test_cl_fix(): assert issubclass(Cl, BaseCl) -@pytest.mark.skipif( - sys.version_info < (3, 7, 6), reason="requires python3.7.6 or newer" -) @unittest.mock.patch( "esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath", autospec=True ) From af108c58f870dd3e2ea05c714324b87e54ef9bff Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Oct 2024 12:29:55 +0100 Subject: [PATCH 07/10] retire Mambaforge so GA can run and run GA --- .github/workflows/run-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 73e15c100c..94963fb038 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - pyproject-toml # run the test only if the PR is to main # turn it on if required #pull_request: @@ -52,7 +53,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt @@ -90,7 +90,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt From b2092809d1b54d761f695b054608a479da168db0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Oct 2024 12:47:04 +0100 Subject: [PATCH 08/10] unrun GA --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 94963fb038..743ac00f45 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - pyproject-toml # run the test only if the PR is to main # turn it on if required #pull_request: From 51f540aee659a53ad8469632c21989d00cd05e4e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 16 Oct 2024 15:43:45 +0200 Subject: [PATCH 09/10] Fixes for ruff B905 --- esmvalcore/_main.py | 1 + esmvalcore/_recipe/check.py | 2 +- esmvalcore/_recipe/from_datasets.py | 2 +- esmvalcore/_task.py | 7 +++++-- esmvalcore/preprocessor/_multimodel.py | 4 ++-- esmvalcore/preprocessor/_time.py | 5 ++++- esmvalcore/preprocessor/_volume.py | 5 ++++- tests/integration/esgf/test_search_download.py | 4 ++-- tests/sample_data/multimodel_statistics/test_multimodel.py | 2 +- tests/unit/esgf/test_search.py | 2 +- tests/unit/preprocessor/_multimodel/test_multimodel.py | 2 +- tests/unit/preprocessor/_regrid/__init__.py | 2 ++ .../preprocessor/_regrid/test_extract_regional_grid.py | 4 ++-- tests/unit/preprocessor/_time/test_time.py | 4 +++- tests/unit/task/test_diagnostic_task.py | 2 +- 15 files changed, 31 insertions(+), 17 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 6978d50f0c..30e2668bdb 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -579,6 +579,7 @@ def _get_config_info(cli_config_dir): zip( config_dirs, _get_all_config_sources(cli_config_dir), + strict=False, ) ) diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 9a26ef4561..5001a5d371 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -171,7 +171,7 @@ def _group_years(years): ends.append(year) ranges = [] - for start, end in zip(starts, ends): + for start, end in zip(starts, ends, strict=False): ranges.append(f"{start}" if start == end else f"{start}-{end}") return ", ".join(ranges) diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index 60384c8026..f68bd9e096 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -219,7 +219,7 @@ def _group_ensemble_names(ensemble_names: Iterable[str]) -> list[str]: groups = [] for ensemble_range in ensemble_ranges: txt = "" - for name, value in zip("ripf", ensemble_range): + for name, value in zip("ripf", ensemble_range, strict=False): txt += name if value[0] == value[1]: txt += f"{value[0]}" diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index cbb53e2691..2a908583b6 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -109,9 +109,12 @@ def _get_resource_usage(process, start_time, children=True): continue # Create and yield log entry - entries = [sum(entry) for entry in zip(*cache.values())] + entries = [sum(entry) for entry in zip(*cache.values(), strict=False)] entries.insert(0, time.time() - start_time) - entries = [round(entry, p) for entry, p in zip(entries, precision)] + entries = [ + round(entry, p) + for entry, p in zip(entries, precision, strict=False) + ] entries.insert(0, datetime.datetime.utcnow()) max_memory = max(max_memory, entries[4]) yield (fmt.format(*entries), max_memory) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index b790b45117..56cea1e936 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -82,13 +82,13 @@ def _unify_time_coordinates(cubes): # monthly data dates = [ datetime(year, month, 15, 0, 0, 0) - for year, month in zip(years, months) + for year, month in zip(years, months, strict=False) ] elif 0 not in np.diff(days): # daily data dates = [ datetime(year, month, day, 0, 0, 0) - for year, month, day in zip(years, months, days) + for year, month, day in zip(years, months, days, strict=False) ] if coord.units != t_unit: logger.warning( diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 062cdf0ba2..a0e8c9e54d 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -672,7 +672,10 @@ def spans_full_season(cube: Cube) -> list[bool]: seasons = cube.coord("clim_season").points tar_days = [(len(sea) * 29, len(sea) * 31) for sea in seasons] - return [dt[0] <= dn <= dt[1] for dn, dt in zip(num_days, tar_days)] + return [ + dt[0] <= dn <= dt[1] + for dn, dt in zip(num_days, tar_days, strict=False) + ] full_seasons = spans_full_season(result) result = result[full_seasons] diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 169dcb3bba..8d56d7b51a 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -510,7 +510,10 @@ def extract_transect( ) for dim_name, dim_cut, coord in zip( - ["latitude", "longitude"], [latitude, longitude], [lats, lons] + ["latitude", "longitude"], + [latitude, longitude], + [lats, lons], + strict=False, ): # #### # Look for the first coordinate. diff --git a/tests/integration/esgf/test_search_download.py b/tests/integration/esgf/test_search_download.py index 65f17b3ef1..5029d75b42 100644 --- a/tests/integration/esgf/test_search_download.py +++ b/tests/integration/esgf/test_search_download.py @@ -170,7 +170,7 @@ def test_mock_search(variable, mocker): assert False, "Wrote expected results, please check." assert len(files) == len(expected_files) - for found_file, expected in zip(files, expected_files): + for found_file, expected in zip(files, expected_files, strict=False): assert found_file.name == expected["name"] assert found_file.local_file(Path()) == Path(expected["local_file"]) assert found_file.dataset == expected["dataset"] @@ -295,7 +295,7 @@ def test_real_search_many(): ] for variable, files, datasets in zip( - VARIABLES, expected_files, expected_datasets + VARIABLES, expected_files, expected_datasets, strict=False ): result = find_files(**variable) found_files = [file.name for file in result] diff --git a/tests/sample_data/multimodel_statistics/test_multimodel.py b/tests/sample_data/multimodel_statistics/test_multimodel.py index 3a9223c15f..815789674f 100644 --- a/tests/sample_data/multimodel_statistics/test_multimodel.py +++ b/tests/sample_data/multimodel_statistics/test_multimodel.py @@ -33,7 +33,7 @@ def assert_array_almost_equal(this, other, rtol=1e-7): def assert_coords_equal(this: list, other: list): """Assert coords list `this` equals coords list `other`.""" - for this_coord, other_coord in zip(this, other): + for this_coord, other_coord in zip(this, other, strict=False): np.testing.assert_equal(this_coord.points, other_coord.points) assert this_coord.var_name == other_coord.var_name assert this_coord.standard_name == other_coord.standard_name diff --git a/tests/unit/esgf/test_search.py b/tests/unit/esgf/test_search.py index 65cd53f1cc..66d0551a8e 100644 --- a/tests/unit/esgf/test_search.py +++ b/tests/unit/esgf/test_search.py @@ -115,7 +115,7 @@ @pytest.mark.parametrize( - "our_facets, esgf_facets", zip(OUR_FACETS, ESGF_FACETS) + "our_facets, esgf_facets", zip(OUR_FACETS, ESGF_FACETS, strict=False) ) def test_get_esgf_facets(our_facets, esgf_facets): """Test that facet translation by get_esgf_facets works as expected.""" diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index aaa41bec82..de379983df 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -912,7 +912,7 @@ def test_ignore_tas_scalar_height_coord(): tas_2m = generate_cube_from_dates("monthly") tas_1p5m = generate_cube_from_dates("monthly") - for cube, height in zip([tas_2m, tas_1p5m], [2.0, 1.5]): + for cube, height in zip([tas_2m, tas_1p5m], [2.0, 1.5], strict=False): cube.rename("air_temperature") cube.attributes["short_name"] = "tas" cube.add_aux_coord( diff --git a/tests/unit/preprocessor/_regrid/__init__.py b/tests/unit/preprocessor/_regrid/__init__.py index a6869d33cf..d70e229cb1 100644 --- a/tests/unit/preprocessor/_regrid/__init__.py +++ b/tests/unit/preprocessor/_regrid/__init__.py @@ -134,6 +134,7 @@ def _make_cube( for a, name in zip( np.meshgrid(node_data_x, node_data_y), ["longitude", "latitude"], + strict=False, ) ] face_data_x = np.arange(x) + 1 @@ -143,6 +144,7 @@ def _make_cube( for a, name in zip( np.meshgrid(face_data_x, face_data_y), ["longitude", "latitude"], + strict=False, ) ] # Build the face connectivity indices by creating an array of squares diff --git a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py index 15055a7658..6afc85162e 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py +++ b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py @@ -32,7 +32,7 @@ def clear_lru_cache(): "step_latitude", ) PASSING_SPECS = tuple( - dict(zip(SPEC_KEYS, spec)) + dict(zip(SPEC_KEYS, spec, strict=False)) for spec in ( (0, 360, 5, -90, 90, 5), (0, 360, 20, -90, 90, 20), @@ -52,7 +52,7 @@ def clear_lru_cache(): ) FAILING_SPECS = tuple( - dict(zip(SPEC_KEYS, spec)) + dict(zip(SPEC_KEYS, spec, strict=False)) for spec in ( # (0, 360, 5, -90, 90, 5), (0, 360, 5, -90, 180, 5), diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index d3518e348d..a1ba351f3d 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -1798,7 +1798,9 @@ def test_anomalies_preserve_metadata(period, reference, standardize=False): metadata = copy.deepcopy(cube.metadata) result = anomalies(cube, period, reference, standardize=standardize) assert result.metadata == metadata - for coord_cube, coord_res in zip(cube.coords(), result.coords()): + for coord_cube, coord_res in zip( + cube.coords(), result.coords(), strict=False + ): if coord_cube.has_bounds() and coord_res.has_bounds(): assert_array_equal(coord_cube.bounds, coord_res.bounds) assert coord_cube == coord_res diff --git a/tests/unit/task/test_diagnostic_task.py b/tests/unit/task/test_diagnostic_task.py index 78b509b040..ca48e9d0ff 100644 --- a/tests/unit/task/test_diagnostic_task.py +++ b/tests/unit/task/test_diagnostic_task.py @@ -236,7 +236,7 @@ def test_collect_provenance(mocker, diagnostic_task): def assert_warned(log, msgs): """Check that messages have been logged.""" assert len(log.records) == len(msgs) - for msg, record in zip(msgs, log.records): + for msg, record in zip(msgs, log.records, strict=False): for snippet in msg: assert snippet in record.message From 953be1c1cb2a2dd5401f4b016a7340a631e9adfb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 16 Oct 2024 15:53:04 +0200 Subject: [PATCH 10/10] Remove ancient pin on esmpy --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 5cf256c22b..cacd62710b 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - dask-jobqueue - distributed - esgf-pyclient >=0.3.1 - - esmpy !=8.1.0 + - esmpy - filelock - fiona - fire diff --git a/pyproject.toml b/pyproject.toml index 2e43a26339..a000674ef1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "dask-jobqueue", "esgf-pyclient>=0.3.1", "esmf-regrid>=0.11.0", - "esmpy!=8.1.0", # not on PyPI + "esmpy", # not on PyPI "filelock", "fiona", "fire",