From 9d598f199806ad2f16cd826b4108cbba5f687952 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Mon, 31 Jan 2022 17:23:53 +0000 Subject: [PATCH 1/7] 3.2 version and whats new (#4559) --- docs/src/whatsnew/{dev.rst => 3.2.rst} | 9 +- docs/src/whatsnew/dev.rst.template | 112 ------------------------- docs/src/whatsnew/index.rst | 2 +- docs/src/whatsnew/latest.rst | 2 +- lib/iris/__init__.py | 2 +- 5 files changed, 7 insertions(+), 120 deletions(-) rename docs/src/whatsnew/{dev.rst => 3.2.rst} (98%) delete mode 100644 docs/src/whatsnew/dev.rst.template diff --git a/docs/src/whatsnew/dev.rst b/docs/src/whatsnew/3.2.rst similarity index 98% rename from docs/src/whatsnew/dev.rst rename to docs/src/whatsnew/3.2.rst index e2d4c2bc0b..c78e1283d6 100644 --- a/docs/src/whatsnew/dev.rst +++ b/docs/src/whatsnew/3.2.rst @@ -1,13 +1,13 @@ .. include:: ../common_links.inc -|iris_version| |build_date| [unreleased] -**************************************** +v3.2 (31 Jan 2022) [unreleased] +******************************* This document explains the changes made to Iris for this release (:doc:`View all changes `.) -.. dropdown:: :opticon:`report` |iris_version| Release Highlights +.. dropdown:: :opticon:`report` v3.2.0 Release Highlights :container: + shadow :title: text-primary text-center font-weight-bold :body: bg-light @@ -18,8 +18,7 @@ This document explains the changes made to Iris for this release * We've added experimental support for :ref:`Meshes `, which can now be loaded and - attached to a cube. Mesh support is based on the based on `CF-UGRID`_ - model. + attached to a cube. Mesh support is based on the `CF-UGRID`_ model. * We've also dropped support for ``Python 3.7``. And finally, get in touch with us on :issue:`GitHub` if you have diff --git a/docs/src/whatsnew/dev.rst.template b/docs/src/whatsnew/dev.rst.template deleted file mode 100644 index 79c578ca65..0000000000 --- a/docs/src/whatsnew/dev.rst.template +++ /dev/null @@ -1,112 +0,0 @@ -.. include:: ../common_links.inc - -|iris_version| |build_date| [unreleased] -**************************************** - -This document explains the changes made to Iris for this release -(:doc:`View all changes `.) - - -.. dropdown:: :opticon:`report` |iris_version| Release Highlights - :container: + shadow - :title: text-primary text-center font-weight-bold - :body: bg-light - :animate: fade-in - :open: - - The highlights for this major/minor release of Iris include: - - * N/A - - And finally, get in touch with us on :issue:`GitHub` if you have - any issues or feature requests for improving Iris. Enjoy! - - -NOTE: section below is a template for bugfix patches -==================================================== - (Please remove this section when creating an initial 'latest.rst') - -v3.X.X (DD MMM YYYY) -==================== - -.. dropdown:: :opticon:`alert` v3.X.X Patches - :container: + shadow - :title: text-primary text-center font-weight-bold - :body: bg-light - :animate: fade-in - - The patches in this release of Iris include: - - #. N/A - -NOTE: section above is a template for bugfix patches -==================================================== - (Please remove this section when creating an initial 'latest.rst') - - - -📢 Announcements -================ - -#. N/A - - -✨ Features -=========== - -#. N/A - - -🐛 Bugs Fixed -============= - -#. N/A - - -💣 Incompatible Changes -======================= - -#. N/A - - -🚀 Performance Enhancements -=========================== - -#. N/A - - -🔥 Deprecations -=============== - -#. N/A - - -🔗 Dependencies -=============== - -#. N/A - - -📚 Documentation -================ - -#. N/A - - -💼 Internal -=========== - -#. N/A - - -.. comment - Whatsnew author names (@github name) in alphabetical order. Note that, - core dev names are automatically included by the common_links.inc: - - - - -.. comment - Whatsnew resources in alphabetical order: - - diff --git a/docs/src/whatsnew/index.rst b/docs/src/whatsnew/index.rst index 51f03e8d8f..f425e649b9 100644 --- a/docs/src/whatsnew/index.rst +++ b/docs/src/whatsnew/index.rst @@ -10,7 +10,7 @@ Iris versions. .. toctree:: :maxdepth: 1 - dev.rst + 3.2.rst 3.1.rst 3.0.rst 2.4.rst diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 56aebe92dd..2bdbea5d85 120000 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -1 +1 @@ -dev.rst \ No newline at end of file +3.2.rst \ No newline at end of file diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index 26f03c0566..aca4e77e88 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -104,7 +104,7 @@ def callback(cube, field, filename): # Iris revision. -__version__ = "3.2.dev0" +__version__ = "3.2.0rc0" # Restrict the names imported when using "from iris import *" __all__ = [ From 8838e23f7461c575adc841fc5c3304c975805d6a Mon Sep 17 00:00:00 2001 From: Bill Little Date: Fri, 4 Feb 2022 10:24:11 +0000 Subject: [PATCH 2/7] update trove classifiers (#4564) --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1d3fb8b7c9..c2d31a5ddb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ classifiers = Operating System :: Unix Programming Language :: Python Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: Implementation :: CPython Topic :: Scientific/Engineering From 611416730970d7603b6e1e4e48dc20c7fb55e2e6 Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Thu, 10 Feb 2022 11:44:56 +0000 Subject: [PATCH 3/7] New tool-agnostic ASV environment management (#4571) * New tool-agnostic ASV env management. * Benchmarking only build the latest Python. * Increase benchmark accuracy by increasing rounds. * Fix ASV rounds mistake. * ASV clearer use of _interpolate_commands. --- benchmarks/asv.conf.json | 23 ++- benchmarks/asv_delegated_conda.py | 208 +++++++++++++++++++++++++ benchmarks/nox_asv_plugin.py | 249 ------------------------------ noxfile.py | 9 +- 4 files changed, 231 insertions(+), 258 deletions(-) create mode 100644 benchmarks/asv_delegated_conda.py delete mode 100644 benchmarks/nox_asv_plugin.py diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 9ea1cdb101..3468b2fca9 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -3,18 +3,25 @@ "project": "scitools-iris", "project_url": "https://github.com/SciTools/iris", "repo": "..", - "environment_type": "nox-conda", + "environment_type": "conda-delegated", "show_commit_url": "http://github.com/scitools/iris/commit/", "benchmark_dir": "./benchmarks", "env_dir": ".asv/env", "results_dir": ".asv/results", "html_dir": ".asv/html", - "plugins": [".nox_asv_plugin"], - // The commit to checkout to first run Nox to set up the environment. - "nox_setup_commit": "HEAD", - // The path of the noxfile's location relative to the project root. - "noxfile_rel_path": "noxfile.py", - // The ``--session`` arg to be used with ``--install-only`` to prep an environment. - "nox_session_name": "tests" + "plugins": [".asv_delegated_conda"], + + // The command(s) that create/update an environment correctly for the + // checked-out commit. + // Interpreted the same as build_command, with following exceptions: + // * No build-time environment variables. + // * Is run in the same environment as the ASV install itself. + "delegated_env_commands": [ + "sed -i 's/_PY_VERSIONS_ALL/_PY_VERSION_LATEST/g' noxfile.py", + "nox --envdir={conf_dir}/.asv/env/nox01 --session=tests --install-only --no-error-on-external-run --verbose" + ], + // The parent directory of the above environment. + // The most recently modified environment in the directory will be used. + "delegated_env_parent": "{conf_dir}/.asv/env/nox01" } diff --git a/benchmarks/asv_delegated_conda.py b/benchmarks/asv_delegated_conda.py new file mode 100644 index 0000000000..250a4e032d --- /dev/null +++ b/benchmarks/asv_delegated_conda.py @@ -0,0 +1,208 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +ASV plug-in providing an alternative :class:`asv.plugins.conda.Conda` +subclass that manages the Conda environment via custom user scripts. + +""" + +from os import environ +from os.path import getmtime +from pathlib import Path +from shutil import copy2, copytree, rmtree +from tempfile import TemporaryDirectory + +from asv import util as asv_util +from asv.config import Config +from asv.console import log +from asv.plugins.conda import Conda +from asv.repo import Repo + + +class CondaDelegated(Conda): + """ + Manage a Conda environment using custom user scripts, run at each commit. + + Ignores user input variations - ``matrix`` / ``pythons`` / + ``conda_environment_file``, since environment is being managed outside ASV. + + Original environment creation behaviour is inherited, but upon checking out + a commit the custom script(s) are run and the original environment is + replaced with a symlink to the custom environment. This arrangement is then + re-used in subsequent runs. + + """ + + tool_name = "conda-delegated" + + def __init__( + self, + conf: Config, + python: str, + requirements: dict, + tagged_env_vars: dict, + ) -> None: + """ + Parameters + ---------- + conf : Config instance + + python : str + Version of Python. Must be of the form "MAJOR.MINOR". + + requirements : dict + Dictionary mapping a PyPI package name to a version + identifier string. + + tagged_env_vars : dict + Environment variables, tagged for build vs. non-build + + """ + ignored = ["`python`"] + if requirements: + ignored.append("`requirements`") + if tagged_env_vars: + ignored.append("`tagged_env_vars`") + if conf.conda_environment_file: + ignored.append("`conda_environment_file`") + message = ( + f"Ignoring ASV setting(s): {', '.join(ignored)}. Benchmark " + "environment management is delegated to third party script(s)." + ) + log.warning(message) + requirements = {} + tagged_env_vars = {} + conf.conda_environment_file = None + + super().__init__(conf, python, requirements, tagged_env_vars) + self._update_info() + + self._env_commands = self._interpolate_commands( + conf.delegated_env_commands + ) + # Again using _interpolate_commands to get env parent path - allows use + # of the same ASV env variables. + env_parent_interpolated = self._interpolate_commands( + conf.delegated_env_parent + ) + # Returns list of tuples, we just want the first. + env_parent_first = env_parent_interpolated[0] + # The 'command' is the first item in the returned tuple. + env_parent_string = " ".join(env_parent_first[0]) + self._delegated_env_parent = Path(env_parent_string).resolve() + + @property + def name(self): + """Get a name to uniquely identify this environment.""" + return asv_util.sanitize_filename(self.tool_name) + + def _update_info(self) -> None: + """Make sure class properties reflect the actual environment being used.""" + # Follow symlink if it has been created. + actual_path = Path(self._path).resolve() + self._path = str(actual_path) + + # Get custom environment's Python version if it exists yet. + try: + get_version = ( + "from sys import version_info; " + "print(f'{version_info.major}.{version_info.minor}')" + ) + actual_python = self.run(["-c", get_version]) + self._python = actual_python + except OSError: + pass + + def _prep_env(self) -> None: + """Run the custom environment script(s) and switch to using that environment.""" + message = f"Running delegated environment management for: {self.name}" + log.info(message) + env_path = Path(self._path) + + def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: + """For copying between self._path and a temporary cache.""" + asv_files = list(src_parent.glob("asv*")) + # build_root_path.name usually == "project" . + asv_files += [src_parent / Path(self._build_root).name] + for src_path in asv_files: + dst_path = dst_parent / src_path.name + if not dst_path.exists(): + # Only caching in case the environment has been rebuilt. + # If the dst_path already exists: rebuilding hasn't + # happened. Also a non-issue when copying in the reverse + # direction because the cache dir is temporary. + if src_path.is_dir(): + func = copytree + else: + func = copy2 + func(src_path, dst_path) + + with TemporaryDirectory(prefix="delegated_asv_cache_") as asv_cache: + asv_cache_path = Path(asv_cache) + # Cache all of ASV's files as delegated command may remove and + # re-build the environment. + copy_asv_files(env_path.resolve(), asv_cache_path) + + # Adapt the build_dir to the cache location. + build_root_path = Path(self._build_root) + build_dir_original = build_root_path / self._repo_subdir + build_dir_subpath = build_dir_original.relative_to( + build_root_path.parent + ) + build_dir = asv_cache_path / build_dir_subpath + + # Run the script(s) for delegated environment creation/updating. + # (An adaptation of self._interpolate_and_run_commands). + for command, env, return_codes, cwd in self._env_commands: + local_envs = dict(environ) + local_envs.update(env) + if cwd is None: + cwd = str(build_dir) + _ = asv_util.check_output( + command, + timeout=self._install_timeout, + cwd=cwd, + env=local_envs, + valid_return_codes=return_codes, + ) + + # Replace the env that ASV created with a symlink to the env + # created/updated by the custom script. + delegated_env_path = sorted( + self._delegated_env_parent.glob("*"), + key=getmtime, + reverse=True, + )[0] + if env_path.resolve() != delegated_env_path: + try: + env_path.unlink(missing_ok=True) + except IsADirectoryError: + rmtree(env_path) + env_path.symlink_to( + delegated_env_path, target_is_directory=True + ) + + # Check that environment exists. + try: + env_path.resolve(strict=True) + except FileNotFoundError: + message = f"Path does not resolve to environment: {env_path}" + log.error(message) + raise RuntimeError(message) + + # Restore ASV's files from the cache (if necessary). + copy_asv_files(asv_cache_path, env_path.resolve()) + + # Record new environment information in properties. + self._update_info() + + def checkout_project(self, repo: Repo, commit_hash: str) -> None: + """Check out the working tree of the project at given commit hash.""" + super().checkout_project(repo, commit_hash) + self._prep_env() + log.info( + f"Environment {self.name} updated to spec at {commit_hash[:8]}" + ) diff --git a/benchmarks/nox_asv_plugin.py b/benchmarks/nox_asv_plugin.py deleted file mode 100644 index 6c9ce14272..0000000000 --- a/benchmarks/nox_asv_plugin.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -ASV plug-in providing an alternative ``Environment`` subclass, which uses Nox -for environment management. - -""" -from importlib.util import find_spec -from pathlib import Path -from shutil import copy2, copytree -from tempfile import TemporaryDirectory - -from asv import util as asv_util -from asv.config import Config -from asv.console import log -from asv.environment import get_env_name -from asv.plugins.conda import Conda, _find_conda -from asv.repo import Repo, get_repo - - -class NoxConda(Conda): - """ - Manage a Conda environment using Nox, updating environment at each commit. - - Defers environment management to the project's noxfile, which must be able - to create/update the benchmarking environment using ``nox --install-only``, - with the ``--session`` specified in ``asv.conf.json.nox_session_name``. - - Notes - ----- - If not all benchmarked commits support this use of Nox: the plugin will - need to be modified to prep the environment in other ways. - - """ - - tool_name = "nox-conda" - - @classmethod - def matches(cls, python: str) -> bool: - """Used by ASV to work out if this type of environment can be used.""" - result = find_spec("nox") is not None - if result: - result = super().matches(python) - - if result: - message = ( - f"NOTE: ASV env match check incomplete. Not possible to know " - f"if selected Nox session (asv.conf.json.nox_session_name) is " - f"compatible with ``--python={python}`` until project is " - f"checked out." - ) - log.warning(message) - - return result - - def __init__(self, conf: Config, python: str, requirements: dict) -> None: - """ - Parameters - ---------- - conf: Config instance - - python : str - Version of Python. Must be of the form "MAJOR.MINOR". - - requirements : dict - Dictionary mapping a PyPI package name to a version - identifier string. - - """ - from nox.sessions import _normalize_path - - # Need to checkout the project BEFORE the benchmark run - to access a noxfile. - self.project_temp_checkout = TemporaryDirectory( - prefix="nox_asv_checkout_" - ) - repo = get_repo(conf) - repo.checkout(self.project_temp_checkout.name, conf.nox_setup_commit) - self.noxfile_rel_path = conf.noxfile_rel_path - self.setup_noxfile = ( - Path(self.project_temp_checkout.name) / self.noxfile_rel_path - ) - self.nox_session_name = conf.nox_session_name - - # Some duplication of parent code - need these attributes BEFORE - # running inherited code. - self._python = python - self._requirements = requirements - self._env_dir = conf.env_dir - - # Prepare the actual environment path, to override self._path. - nox_envdir = str(Path(self._env_dir).absolute() / self.hashname) - nox_friendly_name = self._get_nox_session_name(python) - self._nox_path = Path(_normalize_path(nox_envdir, nox_friendly_name)) - - # For storing any extra conda requirements from asv.conf.json. - self._extra_reqs_path = self._nox_path / "asv-extra-reqs.yaml" - - super().__init__(conf, python, requirements) - - @property - def _path(self) -> str: - """ - Using a property to override getting and setting in parent classes - - unable to modify parent classes as this is a plugin. - - """ - return str(self._nox_path) - - @_path.setter - def _path(self, value) -> None: - """Enforce overriding of this variable by disabling modification.""" - pass - - @property - def name(self) -> str: - """Overridden to prevent inclusion of user input requirements.""" - return get_env_name(self.tool_name, self._python, {}) - - def _get_nox_session_name(self, python: str) -> str: - nox_cmd_substring = ( - f"--noxfile={self.setup_noxfile} " - f"--session={self.nox_session_name} " - f"--python={python}" - ) - - list_output = asv_util.check_output( - ["nox", "--list", *nox_cmd_substring.split(" ")], - display_error=False, - dots=False, - ) - list_output = list_output.split("\n") - list_matches = list(filter(lambda s: s.startswith("*"), list_output)) - matches_count = len(list_matches) - - if matches_count == 0: - message = f"No Nox sessions found for: {nox_cmd_substring} ." - log.error(message) - raise RuntimeError(message) - elif matches_count > 1: - message = ( - f"Ambiguous - >1 Nox session found for: {nox_cmd_substring} ." - ) - log.error(message) - raise RuntimeError(message) - else: - line = list_matches[0] - session_name = line.split(" ")[1] - assert isinstance(session_name, str) - return session_name - - def _nox_prep_env(self, setup: bool = False) -> None: - message = f"Running Nox environment update for: {self.name}" - log.info(message) - - build_root_path = Path(self._build_root) - env_path = Path(self._path) - - def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: - """For copying between self._path and a temporary cache.""" - asv_files = list(src_parent.glob("asv*")) - # build_root_path.name usually == "project" . - asv_files += [src_parent / build_root_path.name] - for src_path in asv_files: - dst_path = dst_parent / src_path.name - if not dst_path.exists(): - # Only cache-ing in case Nox has rebuilt the env @ - # self._path. If the dst_path already exists: rebuilding - # hasn't happened. Also a non-issue when copying in the - # reverse direction because the cache dir is temporary. - if src_path.is_dir(): - func = copytree - else: - func = copy2 - func(src_path, dst_path) - - with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache: - asv_cache_path = Path(asv_cache) - if setup: - noxfile = self.setup_noxfile - else: - # Cache all of ASV's files as Nox may remove and re-build the environment. - copy_asv_files(env_path, asv_cache_path) - # Get location of noxfile in cache. - noxfile_original = ( - build_root_path / self._repo_subdir / self.noxfile_rel_path - ) - noxfile_subpath = noxfile_original.relative_to( - build_root_path.parent - ) - noxfile = asv_cache_path / noxfile_subpath - - nox_cmd = [ - "nox", - f"--noxfile={noxfile}", - # Place the env in the ASV env directory, instead of the default. - f"--envdir={env_path.parent}", - f"--session={self.nox_session_name}", - f"--python={self._python}", - "--install-only", - "--no-error-on-external-run", - "--verbose", - ] - - _ = asv_util.check_output(nox_cmd) - if not env_path.is_dir(): - message = f"Expected Nox environment not found: {env_path}" - log.error(message) - raise RuntimeError(message) - - if not setup: - # Restore ASV's files from the cache (if necessary). - copy_asv_files(asv_cache_path, env_path) - - def _setup(self) -> None: - """Used for initial environment creation - mimics parent method where possible.""" - try: - self.conda = _find_conda() - except IOError as e: - raise asv_util.UserError(str(e)) - if find_spec("nox") is None: - raise asv_util.UserError("Module not found: nox") - - message = f"Creating Nox-Conda environment for {self.name} ." - log.info(message) - - try: - self._nox_prep_env(setup=True) - finally: - # No longer need the setup checkout now that the environment has been built. - self.project_temp_checkout.cleanup() - - conda_args, pip_args = self._get_requirements(self.conda) - if conda_args or pip_args: - message = ( - "Ignoring user input package requirements. Benchmark " - "environment management is exclusively performed by Nox." - ) - log.warning(message) - - def checkout_project(self, repo: Repo, commit_hash: str) -> None: - """Check out the working tree of the project at given commit hash.""" - super().checkout_project(repo, commit_hash) - self._nox_prep_env() - log.info( - f"Environment {self.name} updated to spec at {commit_hash[:8]}" - ) diff --git a/noxfile.py b/noxfile.py index 8b23948677..6367b74aef 100755 --- a/noxfile.py +++ b/noxfile.py @@ -328,7 +328,14 @@ def asv_exec(*sub_args: str) -> None: # Else: compare to previous commit. previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1") try: - asv_exec("continuous", "--factor=1.2", previous_commit, "HEAD") + asv_exec( + "continuous", + "--factor=1.2", + previous_commit, + "HEAD", + "--attribute", + "rounds=4", + ) finally: asv_exec("compare", previous_commit, "HEAD") else: From ee9cadc989931dbd93d59495c92ddec88dbf4e68 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Fri, 11 Feb 2022 11:06:20 +0000 Subject: [PATCH 4/7] Fix load_http bug, extend testing, and note to docs (#4580) * Fix opendap bug, add docs and extra testing * Add whats new entry * Update docs/src/whatsnew/3.2.rst Co-authored-by: Bill Little * Add warning box Co-authored-by: Bill Little --- docs/src/whatsnew/3.2.rst | 3 +++ lib/iris/__init__.py | 8 ++++++++ lib/iris/fileformats/netcdf.py | 4 ++-- lib/iris/io/__init__.py | 8 ++++---- lib/iris/tests/test_load.py | 35 +++++++++++++++++++++++++++------- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/src/whatsnew/3.2.rst b/docs/src/whatsnew/3.2.rst index c78e1283d6..9aa6a78846 100644 --- a/docs/src/whatsnew/3.2.rst +++ b/docs/src/whatsnew/3.2.rst @@ -177,6 +177,9 @@ This document explains the changes made to Iris for this release to indicate cloud fraction greater than 7.9 oktas, rather than 7.5 (:issue:`3305`, :pull:`4535`) +#. `@lbdreyer`_ fixed a bug in :class:`iris.io.load_http` which was missing an import + (:pull:`4580`) + 💣 Incompatible Changes ======================= diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index aca4e77e88..95722c69cf 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -44,6 +44,10 @@ standard library function :func:`os.path.expanduser` and module :mod:`fnmatch` for more details. + .. warning:: + + If supplying a URL, only OPeNDAP Data Sources are supported. + * constraints: Either a single constraint, or an iterable of constraints. Each constraint can be either a string, an instance of @@ -287,6 +291,7 @@ def load(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -315,6 +320,7 @@ def load_cube(uris, constraint=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -354,6 +360,7 @@ def load_cubes(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: @@ -399,6 +406,7 @@ def load_raw(uris, constraints=None, callback=None): * uris: One or more filenames/URIs, as a string or :class:`pathlib.PurePath`. + If supplying a URL, only OPeNDAP Data Sources are supported. Kwargs: diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 100ab29daa..dd819fb63f 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -825,12 +825,12 @@ def inner(cf_datavar): def load_cubes(filenames, callback=None, constraints=None): """ - Loads cubes from a list of NetCDF filenames/URLs. + Loads cubes from a list of NetCDF filenames/OPeNDAP URLs. Args: * filenames (string/list): - One or more NetCDF filenames/DAP URLs to load from. + One or more NetCDF filenames/OPeNDAP URLs to load from. Kwargs: diff --git a/lib/iris/io/__init__.py b/lib/iris/io/__init__.py index 034fa4baab..8d5a2e05d2 100644 --- a/lib/iris/io/__init__.py +++ b/lib/iris/io/__init__.py @@ -216,7 +216,7 @@ def load_files(filenames, callback, constraints=None): def load_http(urls, callback): """ - Takes a list of urls and a callback function, and returns a generator + Takes a list of OPeNDAP URLs and a callback function, and returns a generator of Cubes from the given URLs. .. note:: @@ -226,11 +226,11 @@ def load_http(urls, callback): """ # Create default dict mapping iris format handler to its associated filenames + from iris.fileformats import FORMAT_AGENT + handler_map = collections.defaultdict(list) for url in urls: - handling_format_spec = iris.fileformats.FORMAT_AGENT.get_spec( - url, None - ) + handling_format_spec = FORMAT_AGENT.get_spec(url, None) handler_map[handling_format_spec].append(url) # Call each iris format handler with the appropriate filenames diff --git a/lib/iris/tests/test_load.py b/lib/iris/tests/test_load.py index 86ff2f1ece..d21b40ee26 100644 --- a/lib/iris/tests/test_load.py +++ b/lib/iris/tests/test_load.py @@ -12,6 +12,9 @@ import iris.tests as tests # isort:skip import pathlib +from unittest import mock + +import netCDF4 import iris import iris.io @@ -148,19 +151,20 @@ def test_path_object(self): self.assertEqual(len(cubes), 1) -class TestOpenDAP(tests.IrisTest): - def test_load(self): - # Check that calling iris.load_* with a http URI triggers a call to - # ``iris.io.load_http`` +class TestOPeNDAP(tests.IrisTest): + def setUp(self): + self.url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15" - url = "http://geoport.whoi.edu:80/thredds/dodsC/bathy/gom15" + def test_load_http_called(self): + # Check that calling iris.load_* with an http URI triggers a call to + # ``iris.io.load_http`` class LoadHTTPCalled(Exception): pass def new_load_http(passed_urls, *args, **kwargs): self.assertEqual(len(passed_urls), 1) - self.assertEqual(url, passed_urls[0]) + self.assertEqual(self.url, passed_urls[0]) raise LoadHTTPCalled() try: @@ -174,11 +178,28 @@ def new_load_http(passed_urls, *args, **kwargs): iris.load_cubes, ]: with self.assertRaises(LoadHTTPCalled): - fn(url) + fn(self.url) finally: iris.io.load_http = orig + def test_netCDF_Dataset_call(self): + # Check that load_http calls netCDF4.Dataset and supplies the expected URL. + + # To avoid making a request to an OPeNDAP server in a test, instead + # mock the call to netCDF.Dataset so that it returns a dataset for a + # local file. + filename = tests.get_data_path( + ("NetCDF", "global", "xyt", "SMALL_total_column_co2.nc") + ) + fake_dataset = netCDF4.Dataset(filename) + + with mock.patch( + "netCDF4.Dataset", return_value=fake_dataset + ) as dataset_loader: + next(iris.io.load_http([self.url], callback=None)) + dataset_loader.assert_called_with(self.url, mode="r") + if __name__ == "__main__": tests.main() From 683ed1879b4d43219875cca67467c17fad82b3f4 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Tue, 15 Feb 2022 13:57:37 +0000 Subject: [PATCH 5/7] Finalise whatsnew and version string update (#4588) --- docs/src/whatsnew/3.2.rst | 6 +++--- lib/iris/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/whatsnew/3.2.rst b/docs/src/whatsnew/3.2.rst index 9aa6a78846..ef3764daa5 100644 --- a/docs/src/whatsnew/3.2.rst +++ b/docs/src/whatsnew/3.2.rst @@ -1,7 +1,7 @@ .. include:: ../common_links.inc -v3.2 (31 Jan 2022) [unreleased] -******************************* +v3.2 (15 Feb 2022) +****************** This document explains the changes made to Iris for this release (:doc:`View all changes `.) @@ -351,7 +351,7 @@ This document explains the changes made to Iris for this release #. `@lbdreyer`_ corrected the license PyPI classifier. (:pull:`4435`) -#. `@aaronspring `_ exchanged ``dask`` with +#. `@aaronspring`_ exchanged ``dask`` with ``dask-core`` in testing environments reducing the number of dependencies installed for testing. (:pull:`4434`) diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index 95722c69cf..009a83aed5 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -108,7 +108,7 @@ def callback(cube, field, filename): # Iris revision. -__version__ = "3.2.0rc0" +__version__ = "3.2.0" # Restrict the names imported when using "from iris import *" __all__ = [ From 0365407396f7606b40487c84afe4ff3cc0b9cc45 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Tue, 15 Feb 2022 15:31:22 +0000 Subject: [PATCH 6/7] docs linkcheck skip (#4590) --- docs/src/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/conf.py b/docs/src/conf.py index 19f22e808f..db2cdc3633 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -316,6 +316,7 @@ def _dotv(version): "https://software.ac.uk/how-cite-software", "http://www.esrl.noaa.gov/psd/data/gridded/conventions/cdc_netcdf_standard.shtml", "http://www.nationalarchives.gov.uk/doc/open-government-licence", + "https://www.metoffice.gov.uk/", ] # list of sources to exclude from the build. From f66353f611a1bfc7ee708da32ea330a100bc577f Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Wed, 16 Feb 2022 11:36:58 +0000 Subject: [PATCH 7/7] Add missing commit to v3.2.x and update version number (#4593) * fix trove classifier (#4324) * update version to 3.2. post release Co-authored-by: Bill Little --- lib/iris/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index 009a83aed5..a28a7cd479 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -108,7 +108,7 @@ def callback(cube, field, filename): # Iris revision. -__version__ = "3.2.0" +__version__ = "3.2.0.post0" # Restrict the names imported when using "from iris import *" __all__ = [ diff --git a/setup.cfg b/setup.cfg index c2d31a5ddb..1aabe33d83 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ author = SciTools Developers author_email = scitools-iris-dev@googlegroups.com classifiers = - Development Status :: 5 Production/Stable + Development Status :: 5 - Production/Stable Intended Audience :: Science/Research License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+) Operating System :: MacOS