From e005db914c592dda1907f6c5c1f740f60cd07777 Mon Sep 17 00:00:00 2001 From: TheRed86 Date: Mon, 9 Nov 2020 14:25:38 +0100 Subject: [PATCH 01/21] Define _get_backends_cls function inside apiv2.py to read engines from plugins.py --- xarray/backends/apiv2.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 7e4605c42ce..c7d04714881 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -4,12 +4,24 @@ from . import plugins, zarr from .api import ( _autodetect_engine, - _get_backend_cls, _normalize_path, _protect_dataset_variables_inplace, ) +def _get_backend_cls(engine): + """Select open_dataset method based on current engine""" + + try: + return plugins.ENGINES[engine] + except KeyError: + all_plugins = plugins.ENGINES.keys() + raise ValueError( + "unrecognized engine for open_dataset: {}\n" + "must be one of: {}".format(engine, list(all_plugins)) + ) + + def dataset_from_backend_dataset( ds, filename_or_obj, @@ -233,7 +245,7 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - open_backend_dataset = _get_backend_cls(engine, engines=plugins.ENGINES)[ + open_backend_dataset = _get_backend_cls(engine)[ "open_dataset" ] backend_ds = open_backend_dataset( From 0ae8bbd5ee421a4993978ad3de0b63e27fd39f87 Mon Sep 17 00:00:00 2001 From: TheRed86 Date: Mon, 9 Nov 2020 14:26:41 +0100 Subject: [PATCH 02/21] Read open_backends_dataset_* from entrypoints. --- xarray/backends/plugins.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index a0b1a825600..4bce2ba4526 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -1,17 +1,20 @@ import inspect import typing as T +import entrypoints +import sys -from . import cfgrib_, h5netcdf_, zarr +path = sys.path.copy() +path = set(path) ENGINES: T.Dict[str, T.Dict[str, T.Any]] = { "h5netcdf": { - "open_dataset": h5netcdf_.open_backend_dataset_h5necdf, + "open_dataset": entrypoints.get_single("xarray.backends", "h5netcdf", path=path).load(), }, "zarr": { - "open_dataset": zarr.open_backend_dataset_zarr, + "open_dataset": entrypoints.get_single("xarray.backends", "zarr", path=path).load(), }, "cfgrib": { - "open_dataset": cfgrib_.open_backend_dataset_cfgrib, + "open_dataset": entrypoints.get_single("xarray.backends", "cfgrib", path=path).load(), }, } From df4f3de028c317f936453480294f7aceb90bd368 Mon Sep 17 00:00:00 2001 From: TheRed86 Date: Mon, 9 Nov 2020 14:27:29 +0100 Subject: [PATCH 03/21] Add backend entrypoints in setup.cfg --- setup.cfg | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2a7762fb9f5..b2964423516 100644 --- a/setup.cfg +++ b/setup.cfg @@ -81,6 +81,12 @@ setup_requires = setuptools >= 38.4 setuptools_scm +[options.entry_points] +xarray.backends = + zarr = xarray.backends.zarr:open_backend_dataset_zarr + h5netcdf = xarray.backends.h5netcdf_:open_backend_dataset_h5necdf + cfgrib = xarray.backends.cfgrib_:open_backend_dataset_cfgrib + [options.extras_require] io = netCDF4 From ea08b03044c39cd6ab521829e13ad74b58d34c85 Mon Sep 17 00:00:00 2001 From: TheRed86 Date: Mon, 9 Nov 2020 14:31:09 +0100 Subject: [PATCH 04/21] Pass apiv2.py isort and black formatting tests. --- xarray/backends/apiv2.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index c7d04714881..85b315a6b33 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -2,11 +2,7 @@ from ..core.utils import is_remote_uri from . import plugins, zarr -from .api import ( - _autodetect_engine, - _normalize_path, - _protect_dataset_variables_inplace, -) +from .api import _autodetect_engine, _normalize_path, _protect_dataset_variables_inplace def _get_backend_cls(engine): @@ -245,9 +241,7 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - open_backend_dataset = _get_backend_cls(engine)[ - "open_dataset" - ] + open_backend_dataset = _get_backend_cls(engine)["open_dataset"] backend_ds = open_backend_dataset( filename_or_obj, drop_variables=drop_variables, From 131a2b6bfb351aed2488bec3a6ee794f0d991f75 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 11 Nov 2020 16:20:26 +0100 Subject: [PATCH 05/21] add dependencies --- .binder/environment.yml | 1 + ci/requirements/py36-bare-minimum.yml | 1 + ci/requirements/py36-min-all-deps.yml | 1 + ci/requirements/py36-min-nep18.yml | 1 + ci/requirements/py36.yml | 1 + ci/requirements/py37-windows.yml | 1 + ci/requirements/py37.yml | 1 + ci/requirements/py38-all-but-dask.yml | 1 + ci/requirements/py38.yml | 1 + setup.cfg | 1 + 10 files changed, 10 insertions(+) diff --git a/.binder/environment.yml b/.binder/environment.yml index 6fd5829c5e6..d394be9a3e3 100644 --- a/.binder/environment.yml +++ b/.binder/environment.yml @@ -13,6 +13,7 @@ dependencies: - dask - distributed - dask_labextension + - entrypoints - h5netcdf - h5py - hdf5 diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index aaba5366f67..e91971fbc16 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -4,6 +4,7 @@ channels: dependencies: - python=3.6 - coveralls + - entrypoints - pip - pytest - pytest-cov diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 3ca7b2581e4..9616f784aca 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -17,6 +17,7 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 + - entrypoints - flake8 - h5netcdf=0.7 - h5py=2.9 # Policy allows for 2.10, but it's a conflict-fest diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml index 14982c1d5e7..807c072ae85 100644 --- a/ci/requirements/py36-min-nep18.yml +++ b/ci/requirements/py36-min-nep18.yml @@ -8,6 +8,7 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 + - entrypoints - numpy=1.17 - pandas=0.25 - pint=0.15 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index 7c06bcb88f0..e08ee72a6e9 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -13,6 +13,7 @@ dependencies: - coveralls - dask - distributed + - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 64c73c04c82..12fde9d3ff4 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -13,6 +13,7 @@ dependencies: - coveralls - dask - distributed + - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index c9702deb4bf..2d005cb6fd7 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -13,6 +13,7 @@ dependencies: - coveralls - dask - distributed + - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py38-all-but-dask.yml b/ci/requirements/py38-all-but-dask.yml index 547418ada66..ff89e95e5dc 100644 --- a/ci/requirements/py38-all-but-dask.yml +++ b/ci/requirements/py38-all-but-dask.yml @@ -11,6 +11,7 @@ dependencies: - cfgrib - cftime - coveralls + - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py38.yml b/ci/requirements/py38.yml index b80b4fde6fd..f5caa2fa312 100644 --- a/ci/requirements/py38.yml +++ b/ci/requirements/py38.yml @@ -13,6 +13,7 @@ dependencies: - coveralls - dask - distributed + - entrypoints - flake8 - h5netcdf - h5py diff --git a/setup.cfg b/setup.cfg index b2964423516..b127d27a50e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,6 +77,7 @@ install_requires = numpy >= 1.15 pandas >= 0.25 setuptools >= 38.4 # For pkg_resources + entrypoints >= 0.3 setup_requires = setuptools >= 38.4 setuptools_scm From e2bdeaaf5621529ce26a67d5ea841a4edc35c077 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 12 Nov 2020 13:08:24 +0100 Subject: [PATCH 06/21] add backend entrypoints and check on conflicts --- setup.cfg | 6 +- xarray/backends/__init__.py | 1 + xarray/backends/apiv2.py | 32 ++++----- xarray/backends/cfgrib_.py | 2 + xarray/backends/h5netcdf_.py | 5 +- xarray/backends/plugins.py | 124 ++++++++++++++++++++++++++--------- xarray/backends/zarr.py | 2 + 7 files changed, 118 insertions(+), 54 deletions(-) diff --git a/setup.cfg b/setup.cfg index b127d27a50e..54e629e8e75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,9 +84,9 @@ setup_requires = [options.entry_points] xarray.backends = - zarr = xarray.backends.zarr:open_backend_dataset_zarr - h5netcdf = xarray.backends.h5netcdf_:open_backend_dataset_h5necdf - cfgrib = xarray.backends.cfgrib_:open_backend_dataset_cfgrib + zarr = xarray.backends.zarr:zarr_backend + h5netcdf = xarray.backends.h5netcdf_:h5netcdf_backend + cfgrib = xarray.backends.cfgrib_:cfgrib_backend [options.extras_require] io = diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index 2a769b1335e..63cd3b66982 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -14,6 +14,7 @@ from .pynio_ import NioDataStore from .scipy_ import ScipyDataStore from .zarr import ZarrStore +from .plugins import ENGINES __all__ = [ "AbstractDataStore", diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 85b315a6b33..eaca0d148c6 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -2,20 +2,12 @@ from ..core.utils import is_remote_uri from . import plugins, zarr -from .api import _autodetect_engine, _normalize_path, _protect_dataset_variables_inplace - - -def _get_backend_cls(engine): - """Select open_dataset method based on current engine""" - - try: - return plugins.ENGINES[engine] - except KeyError: - all_plugins = plugins.ENGINES.keys() - raise ValueError( - "unrecognized engine for open_dataset: {}\n" - "must be one of: {}".format(engine, list(all_plugins)) - ) +from .api import ( + _autodetect_engine, + _get_backend_cls, + _normalize_path, + _protect_dataset_variables_inplace, +) def dataset_from_backend_dataset( @@ -80,11 +72,10 @@ def dataset_from_backend_dataset( return ds2 -def resolve_decoders_kwargs(decode_cf, engine, **decoders): - signature = plugins.ENGINES[engine]["signature"] +def resolve_decoders_kwargs(decode_cf, open_backend_dataset_parameters, **decoders): if decode_cf is False: for d in decoders: - if d in signature: + if d in open_backend_dataset_parameters: decoders[d] = False return {k: v for k, v in decoders.items() if v is not None} @@ -227,9 +218,13 @@ def open_dataset( if engine is None: engine = _autodetect_engine(filename_or_obj) + backend = _get_backend_cls(engine, engines=plugins.ENGINES) + open_backend_dataset = backend["open_dataset"] + open_backend_dataset_parameters = backend["open_dataset_parameters"] + decoders = resolve_decoders_kwargs( decode_cf, - engine=engine, + open_backend_dataset_parameters=open_backend_dataset_parameters, mask_and_scale=mask_and_scale, decode_times=decode_times, decode_timedelta=decode_timedelta, @@ -241,7 +236,6 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - open_backend_dataset = _get_backend_cls(engine)["open_dataset"] backend_ds = open_backend_dataset( filename_or_obj, drop_variables=drop_variables, diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 80b3183f197..0ee033191aa 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -126,3 +126,5 @@ def open_backend_dataset_cfgrib( ds.encoding = encoding return ds + +cfgrib_backend = {'open_dataset': open_backend_dataset_cfgrib} \ No newline at end of file diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 9d013e3e46e..32699eb08ca 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -325,7 +325,7 @@ def close(self, **kwargs): self._manager.close(**kwargs) -def open_backend_dataset_h5necdf( +def open_backend_dataset_h5netcdf( filename_or_obj, *, mask_and_scale=True, @@ -374,3 +374,6 @@ def open_backend_dataset_h5necdf( ds.encoding = encoding return ds + + +h5netcdf_backend = {'open_dataset': open_backend_dataset_h5netcdf} \ No newline at end of file diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 4bce2ba4526..d1773ff40fd 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -1,34 +1,96 @@ import inspect +import itertools import typing as T +import warnings + import entrypoints -import sys - -path = sys.path.copy() -path = set(path) - -ENGINES: T.Dict[str, T.Dict[str, T.Any]] = { - "h5netcdf": { - "open_dataset": entrypoints.get_single("xarray.backends", "h5netcdf", path=path).load(), - }, - "zarr": { - "open_dataset": entrypoints.get_single("xarray.backends", "zarr", path=path).load(), - }, - "cfgrib": { - "open_dataset": entrypoints.get_single("xarray.backends", "cfgrib", path=path).load(), - }, -} - - -for engine in ENGINES.values(): - if "signature" not in engine: - parameters = inspect.signature(engine["open_dataset"]).parameters - for name, param in parameters.items(): - if param.kind in ( - inspect.Parameter.VAR_KEYWORD, - inspect.Parameter.VAR_POSITIONAL, - ): - raise TypeError( - f'All the parameters in {engine["open_dataset"]!r} signature should be explicit. ' - "*args and **kwargs is not supported" - ) - engine["signature"] = set(parameters) + + +def get_entrypoint_id(entrypoint): + name = entrypoint.name + obj = entrypoint.object_name or "" + module = entrypoint.module_name + return (name, f"{module}:{obj}") + + +def filter_unique_ids(backend_entrypoints): + entrypoints_id = set() + backend_entrypoints_unique = [] + for entrypoint in backend_entrypoints: + id = get_entrypoint_id(entrypoint) + if id not in entrypoints_id: + backend_entrypoints_unique.append(entrypoint) + entrypoints_id.add(id) + return backend_entrypoints_unique + + +def warning_on_entrypoints_conflict( + backend_entrypoints, + backend_entrypoints_all +): + + # sort and group entrypoints by name + key_name = lambda ep: ep.name + backend_entrypoints_all_unique_ids = sorted( + backend_entrypoints_all, + key=key_name + ) + backend_entrypoints_ids_grouped = itertools.groupby( + backend_entrypoints_all_unique_ids, + key=key_name + ) + + # check if there are multiple entrypoints for the same name + for name, matches in backend_entrypoints_ids_grouped: + matches = list(matches) + matches_len = len(matches) + if matches_len > 1: + selected_entrypoint = backend_entrypoints[name] + warnings.warn( + f"\nFound {matches_len} entrypoints " + f"for the engine name {name}:\n {matches}.\n" + f"It will be used: {selected_entrypoint}.", + ) + + +def detect_parameters(open_dataset): + signature = inspect.signature(open_dataset) + parameters = signature.parameters + for name, param in parameters.items(): + if param.kind in ( + inspect.Parameter.VAR_KEYWORD, + inspect.Parameter.VAR_POSITIONAL, + ): + raise TypeError( + f'All the parameters in {open_dataset!r} signature should be explicit. ' + "*args and **kwargs is not supported" + ) + return set(parameters) + + +def detect_engines(): + backend_entrypoints = entrypoints.get_group_named('xarray.backends') + backend_entrypoints_all = entrypoints.get_group_all('xarray.backends') + backend_entrypoints_all = filter_unique_ids(backend_entrypoints_all) + + if len(backend_entrypoints_all) != len(backend_entrypoints): + warning_on_entrypoints_conflict( + backend_entrypoints, + backend_entrypoints_all + ) + + engines: T.Dict[str, T.Dict[str, T.Any]] = {} + for name, backend in backend_entrypoints.items(): + backend = backend.load() + engines[name] = backend + + open_dataset = backend["open_dataset"] + if "signature" in backend: + pass + backend['open_dataset_parameters'] = detect_parameters(open_dataset) + return engines + + +ENGINES = detect_engines() + + diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 9827c345239..b5abc9b7d18 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -741,3 +741,5 @@ def open_backend_dataset_zarr( ds.encoding = encoding return ds + +zarr_backend = {'open_dataset': open_backend_dataset_zarr} \ No newline at end of file From 2f99e9ca24136558d556575366af757c6049aab7 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 12 Nov 2020 13:15:33 +0100 Subject: [PATCH 07/21] black --- xarray/backends/plugins.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index d1773ff40fd..e008a8fc094 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -24,20 +24,13 @@ def filter_unique_ids(backend_entrypoints): return backend_entrypoints_unique -def warning_on_entrypoints_conflict( - backend_entrypoints, - backend_entrypoints_all -): +def warning_on_entrypoints_conflict(backend_entrypoints, backend_entrypoints_all): # sort and group entrypoints by name key_name = lambda ep: ep.name - backend_entrypoints_all_unique_ids = sorted( - backend_entrypoints_all, - key=key_name - ) + backend_entrypoints_all_unique_ids = sorted(backend_entrypoints_all, key=key_name) backend_entrypoints_ids_grouped = itertools.groupby( - backend_entrypoints_all_unique_ids, - key=key_name + backend_entrypoints_all_unique_ids, key=key_name ) # check if there are multiple entrypoints for the same name @@ -62,22 +55,19 @@ def detect_parameters(open_dataset): inspect.Parameter.VAR_POSITIONAL, ): raise TypeError( - f'All the parameters in {open_dataset!r} signature should be explicit. ' + f"All the parameters in {open_dataset!r} signature should be explicit. " "*args and **kwargs is not supported" ) return set(parameters) def detect_engines(): - backend_entrypoints = entrypoints.get_group_named('xarray.backends') - backend_entrypoints_all = entrypoints.get_group_all('xarray.backends') + backend_entrypoints = entrypoints.get_group_named("xarray.backends") + backend_entrypoints_all = entrypoints.get_group_all("xarray.backends") backend_entrypoints_all = filter_unique_ids(backend_entrypoints_all) if len(backend_entrypoints_all) != len(backend_entrypoints): - warning_on_entrypoints_conflict( - backend_entrypoints, - backend_entrypoints_all - ) + warning_on_entrypoints_conflict(backend_entrypoints, backend_entrypoints_all) engines: T.Dict[str, T.Dict[str, T.Any]] = {} for name, backend in backend_entrypoints.items(): @@ -87,10 +77,8 @@ def detect_engines(): open_dataset = backend["open_dataset"] if "signature" in backend: pass - backend['open_dataset_parameters'] = detect_parameters(open_dataset) + backend["open_dataset_parameters"] = detect_parameters(open_dataset) return engines ENGINES = detect_engines() - - From 9c14a02ad600ac2617f9fdb615877e0d0db16a34 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 12 Nov 2020 16:11:06 +0100 Subject: [PATCH 08/21] removed global variable EMGINES add class for entrypointys --- xarray/backends/__init__.py | 3 +-- xarray/backends/api.py | 2 +- xarray/backends/apiv2.py | 6 +++--- xarray/backends/cfgrib_.py | 4 +++- xarray/backends/h5netcdf_.py | 4 +++- xarray/backends/plugins.py | 25 ++++++++++++++----------- xarray/backends/zarr.py | 3 ++- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index 63cd3b66982..190f39e48f3 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -14,8 +14,7 @@ from .pynio_ import NioDataStore from .scipy_ import ScipyDataStore from .zarr import ZarrStore -from .plugins import ENGINES - +from .plugins import detect_engines __all__ = [ "AbstractDataStore", "FileManager", diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 0b9b5046cb9..06467239948 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -437,7 +437,7 @@ def open_dataset( kwargs = locals().copy() from . import apiv2, plugins - if engine in plugins.ENGINES: + if engine in plugins.detect_engines(): return apiv2.open_dataset(**kwargs) if autoclose is not None: diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index eaca0d148c6..3d04f1af200 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -218,9 +218,9 @@ def open_dataset( if engine is None: engine = _autodetect_engine(filename_or_obj) - backend = _get_backend_cls(engine, engines=plugins.ENGINES) - open_backend_dataset = backend["open_dataset"] - open_backend_dataset_parameters = backend["open_dataset_parameters"] + backend = _get_backend_cls(engine, engines=plugins.detect_engines()) + open_backend_dataset = backend.open_dataset + open_backend_dataset_parameters = backend.open_dataset_parameters decoders = resolve_decoders_kwargs( decode_cf, diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 0ee033191aa..bed539433aa 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -7,6 +7,8 @@ from ..core.variable import Variable from .common import AbstractDataStore, BackendArray from .locks import SerializableLock, ensure_lock +from .plugins import BackendEntrypoint + # FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe # in most circumstances. See: @@ -127,4 +129,4 @@ def open_backend_dataset_cfgrib( return ds -cfgrib_backend = {'open_dataset': open_backend_dataset_cfgrib} \ No newline at end of file +cfgrib_backend = BackendEntrypoint(open_dataset=open_backend_dataset_cfgrib) \ No newline at end of file diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 32699eb08ca..4c64b7b8156 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -18,6 +18,8 @@ _get_datatype, _nc4_require_group, ) +from .plugins import BackendEntrypoint + class H5NetCDFArrayWrapper(BaseNetCDF4Array): @@ -376,4 +378,4 @@ def open_backend_dataset_h5netcdf( return ds -h5netcdf_backend = {'open_dataset': open_backend_dataset_h5netcdf} \ No newline at end of file +h5netcdf_backend = BackendEntrypoint(open_dataset=open_backend_dataset_h5netcdf) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index e008a8fc094..1a43abb4de5 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -2,8 +2,15 @@ import itertools import typing as T import warnings +from functools import lru_cache -import entrypoints +import entrypoints # type: ignore + + +class BackendEntrypoint: + def __init__(self, open_dataset, open_dataset_parameters=None): + self.open_dataset = open_dataset + self.open_dataset_parameters = open_dataset_parameters def get_entrypoint_id(entrypoint): @@ -40,9 +47,8 @@ def warning_on_entrypoints_conflict(backend_entrypoints, backend_entrypoints_all if matches_len > 1: selected_entrypoint = backend_entrypoints[name] warnings.warn( - f"\nFound {matches_len} entrypoints " - f"for the engine name {name}:\n {matches}.\n" - f"It will be used: {selected_entrypoint}.", + f"\nFound {matches_len} entrypoints for the engine name {name}:" + f"\n {matches}.\n It will be used: {selected_entrypoint}.", ) @@ -61,6 +67,7 @@ def detect_parameters(open_dataset): return set(parameters) +@lru_cache(maxsize=1) def detect_engines(): backend_entrypoints = entrypoints.get_group_named("xarray.backends") backend_entrypoints_all = entrypoints.get_group_all("xarray.backends") @@ -74,11 +81,7 @@ def detect_engines(): backend = backend.load() engines[name] = backend - open_dataset = backend["open_dataset"] - if "signature" in backend: - pass - backend["open_dataset_parameters"] = detect_parameters(open_dataset) + if backend.open_dataset_parameters is None: + open_dataset = backend.open_dataset + backend.open_dataset_parameters = detect_parameters(open_dataset) return engines - - -ENGINES = detect_engines() diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index b5abc9b7d18..43aa2b4c0cb 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -9,6 +9,7 @@ from ..core.utils import FrozenDict, HiddenKeyDict, close_on_error from ..core.variable import Variable from .common import AbstractWritableDataStore, BackendArray, _encode_variable_name +from .plugins import BackendEntrypoint # need some special secret attributes to tell us the dimensions DIMENSION_KEY = "_ARRAY_DIMENSIONS" @@ -742,4 +743,4 @@ def open_backend_dataset_zarr( return ds -zarr_backend = {'open_dataset': open_backend_dataset_zarr} \ No newline at end of file +zarr_backend = BackendEntrypoint(open_dataset= open_backend_dataset_zarr) \ No newline at end of file From d1849472d05abc55d37dc7de57246e1940f9ece1 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 12 Nov 2020 16:34:39 +0100 Subject: [PATCH 09/21] black isort --- xarray/backends/__init__.py | 3 ++- xarray/backends/cfgrib_.py | 4 ++-- xarray/backends/h5netcdf_.py | 1 - xarray/backends/plugins.py | 4 +++- xarray/backends/zarr.py | 3 ++- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index 190f39e48f3..d0ee8eb3f39 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -9,12 +9,13 @@ from .h5netcdf_ import H5NetCDFStore from .memory import InMemoryDataStore from .netCDF4_ import NetCDF4DataStore +from .plugins import detect_engines from .pseudonetcdf_ import PseudoNetCDFDataStore from .pydap_ import PydapDataStore from .pynio_ import NioDataStore from .scipy_ import ScipyDataStore from .zarr import ZarrStore -from .plugins import detect_engines + __all__ = [ "AbstractDataStore", "FileManager", diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index bed539433aa..cf506d67e96 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -9,7 +9,6 @@ from .locks import SerializableLock, ensure_lock from .plugins import BackendEntrypoint - # FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe # in most circumstances. See: # https://confluence.ecmwf.int/display/ECC/Frequently+Asked+Questions @@ -129,4 +128,5 @@ def open_backend_dataset_cfgrib( return ds -cfgrib_backend = BackendEntrypoint(open_dataset=open_backend_dataset_cfgrib) \ No newline at end of file + +cfgrib_backend = BackendEntrypoint(open_dataset=open_backend_dataset_cfgrib) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 4c64b7b8156..e3539a05fb1 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -21,7 +21,6 @@ from .plugins import BackendEntrypoint - class H5NetCDFArrayWrapper(BaseNetCDF4Array): def get_array(self, needs_lock=True): ds = self.datastore._acquire(needs_lock) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 1a43abb4de5..63e8a6692de 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -4,10 +4,12 @@ import warnings from functools import lru_cache -import entrypoints # type: ignore +import entrypoints # type: ignore class BackendEntrypoint: + __slots__ = ("open_dataset", "open_dataset_parameters") + def __init__(self, open_dataset, open_dataset_parameters=None): self.open_dataset = open_dataset self.open_dataset_parameters = open_dataset_parameters diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 43aa2b4c0cb..51822492331 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -743,4 +743,5 @@ def open_backend_dataset_zarr( return ds -zarr_backend = BackendEntrypoint(open_dataset= open_backend_dataset_zarr) \ No newline at end of file + +zarr_backend = BackendEntrypoint(open_dataset=open_backend_dataset_zarr) From a37d549166000d203b42338bb66c62044e7e5a30 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 12 Nov 2020 17:05:52 +0100 Subject: [PATCH 10/21] add detect_engines in __all__ init.py --- xarray/backends/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index d0ee8eb3f39..d7b73daf9b0 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -30,4 +30,5 @@ "H5NetCDFStore", "ZarrStore", "PseudoNetCDFDataStore", + "detect_engines", ] From 1c49d73199946561bbd42d984d3b5a288338a025 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 19 Nov 2020 16:11:47 +0100 Subject: [PATCH 11/21] removed entrypoints in py36-bare-minimum.yml and py36-min-all-deps.yml --- ci/requirements/py36-bare-minimum.yml | 1 - ci/requirements/py36-min-all-deps.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index e91971fbc16..aaba5366f67 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -4,7 +4,6 @@ channels: dependencies: - python=3.6 - coveralls - - entrypoints - pip - pytest - pytest-cov diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 9616f784aca..3ca7b2581e4 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -17,7 +17,6 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 - - entrypoints - flake8 - h5netcdf=0.7 - h5py=2.9 # Policy allows for 2.10, but it's a conflict-fest From 94451c656615c69d084f2930d04e1c1a3c757aef Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 19 Nov 2020 17:00:46 +0100 Subject: [PATCH 12/21] add entrypoints in IGNORE_DEPS --- ci/min_deps_check.py | 1 + ci/requirements/py36-bare-minimum.yml | 1 + ci/requirements/py36-min-all-deps.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 93d12754365..a60f878ddaf 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -13,6 +13,7 @@ IGNORE_DEPS = { "black", "coveralls", + "entrypoints", "flake8", "hypothesis", "isort", diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index aaba5366f67..e91971fbc16 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -4,6 +4,7 @@ channels: dependencies: - python=3.6 - coveralls + - entrypoints - pip - pytest - pytest-cov diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 3ca7b2581e4..9616f784aca 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -17,6 +17,7 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 + - entrypoints - flake8 - h5netcdf=0.7 - h5py=2.9 # Policy allows for 2.10, but it's a conflict-fest From 5dd4714b0685f23deeba8cddb68d231a0cad7d02 Mon Sep 17 00:00:00 2001 From: aurghs <35919497+aurghs@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:10:30 +0100 Subject: [PATCH 13/21] Plugins test (#20) - replace entrypoints with pkg_resources - add tests --- .binder/environment.yml | 1 - ci/min_deps_check.py | 1 - ci/requirements/py36-bare-minimum.yml | 1 - ci/requirements/py36-min-all-deps.yml | 1 - ci/requirements/py36-min-nep18.yml | 1 - ci/requirements/py36.yml | 1 - ci/requirements/py37-windows.yml | 1 - ci/requirements/py37.yml | 1 - ci/requirements/py38-all-but-dask.yml | 1 - ci/requirements/py38.yml | 1 - setup.cfg | 1 - xarray/backends/__init__.py | 4 +- xarray/backends/api.py | 2 +- xarray/backends/apiv2.py | 20 +++--- xarray/backends/plugins.py | 73 +++++++++----------- xarray/tests/test_plugins.py | 97 +++++++++++++++++++++++++++ 16 files changed, 141 insertions(+), 66 deletions(-) create mode 100644 xarray/tests/test_plugins.py diff --git a/.binder/environment.yml b/.binder/environment.yml index d394be9a3e3..6fd5829c5e6 100644 --- a/.binder/environment.yml +++ b/.binder/environment.yml @@ -13,7 +13,6 @@ dependencies: - dask - distributed - dask_labextension - - entrypoints - h5netcdf - h5py - hdf5 diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index a60f878ddaf..93d12754365 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -13,7 +13,6 @@ IGNORE_DEPS = { "black", "coveralls", - "entrypoints", "flake8", "hypothesis", "isort", diff --git a/ci/requirements/py36-bare-minimum.yml b/ci/requirements/py36-bare-minimum.yml index e91971fbc16..aaba5366f67 100644 --- a/ci/requirements/py36-bare-minimum.yml +++ b/ci/requirements/py36-bare-minimum.yml @@ -4,7 +4,6 @@ channels: dependencies: - python=3.6 - coveralls - - entrypoints - pip - pytest - pytest-cov diff --git a/ci/requirements/py36-min-all-deps.yml b/ci/requirements/py36-min-all-deps.yml index 9616f784aca..3ca7b2581e4 100644 --- a/ci/requirements/py36-min-all-deps.yml +++ b/ci/requirements/py36-min-all-deps.yml @@ -17,7 +17,6 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 - - entrypoints - flake8 - h5netcdf=0.7 - h5py=2.9 # Policy allows for 2.10, but it's a conflict-fest diff --git a/ci/requirements/py36-min-nep18.yml b/ci/requirements/py36-min-nep18.yml index 807c072ae85..14982c1d5e7 100644 --- a/ci/requirements/py36-min-nep18.yml +++ b/ci/requirements/py36-min-nep18.yml @@ -8,7 +8,6 @@ dependencies: - coveralls - dask=2.9 - distributed=2.9 - - entrypoints - numpy=1.17 - pandas=0.25 - pint=0.15 diff --git a/ci/requirements/py36.yml b/ci/requirements/py36.yml index e08ee72a6e9..7c06bcb88f0 100644 --- a/ci/requirements/py36.yml +++ b/ci/requirements/py36.yml @@ -13,7 +13,6 @@ dependencies: - coveralls - dask - distributed - - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py37-windows.yml b/ci/requirements/py37-windows.yml index 12fde9d3ff4..64c73c04c82 100644 --- a/ci/requirements/py37-windows.yml +++ b/ci/requirements/py37-windows.yml @@ -13,7 +13,6 @@ dependencies: - coveralls - dask - distributed - - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py37.yml b/ci/requirements/py37.yml index 2d005cb6fd7..c9702deb4bf 100644 --- a/ci/requirements/py37.yml +++ b/ci/requirements/py37.yml @@ -13,7 +13,6 @@ dependencies: - coveralls - dask - distributed - - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py38-all-but-dask.yml b/ci/requirements/py38-all-but-dask.yml index ff89e95e5dc..547418ada66 100644 --- a/ci/requirements/py38-all-but-dask.yml +++ b/ci/requirements/py38-all-but-dask.yml @@ -11,7 +11,6 @@ dependencies: - cfgrib - cftime - coveralls - - entrypoints - flake8 - h5netcdf - h5py diff --git a/ci/requirements/py38.yml b/ci/requirements/py38.yml index f5caa2fa312..b80b4fde6fd 100644 --- a/ci/requirements/py38.yml +++ b/ci/requirements/py38.yml @@ -13,7 +13,6 @@ dependencies: - coveralls - dask - distributed - - entrypoints - flake8 - h5netcdf - h5py diff --git a/setup.cfg b/setup.cfg index 54e629e8e75..0a82f80ebd6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,7 +77,6 @@ install_requires = numpy >= 1.15 pandas >= 0.25 setuptools >= 38.4 # For pkg_resources - entrypoints >= 0.3 setup_requires = setuptools >= 38.4 setuptools_scm diff --git a/xarray/backends/__init__.py b/xarray/backends/__init__.py index d7b73daf9b0..1500ea5061f 100644 --- a/xarray/backends/__init__.py +++ b/xarray/backends/__init__.py @@ -9,7 +9,7 @@ from .h5netcdf_ import H5NetCDFStore from .memory import InMemoryDataStore from .netCDF4_ import NetCDF4DataStore -from .plugins import detect_engines +from .plugins import list_engines from .pseudonetcdf_ import PseudoNetCDFDataStore from .pydap_ import PydapDataStore from .pynio_ import NioDataStore @@ -30,5 +30,5 @@ "H5NetCDFStore", "ZarrStore", "PseudoNetCDFDataStore", - "detect_engines", + "list_engines", ] diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 06467239948..ccb3c4d52b4 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -437,7 +437,7 @@ def open_dataset( kwargs = locals().copy() from . import apiv2, plugins - if engine in plugins.detect_engines(): + if engine in plugins.list_engines(): return apiv2.open_dataset(**kwargs) if autoclose is not None: diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 3d04f1af200..c469f3fc3cf 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -73,11 +73,12 @@ def dataset_from_backend_dataset( def resolve_decoders_kwargs(decode_cf, open_backend_dataset_parameters, **decoders): - if decode_cf is False: - for d in decoders: - if d in open_backend_dataset_parameters: - decoders[d] = False - return {k: v for k, v in decoders.items() if v is not None} + for d in list(decoders): + if decode_cf is False and d in open_backend_dataset_parameters: + decoders[d] = False + if decoders[d] is None: + decoders.pop(d) + return decoders def open_dataset( @@ -218,13 +219,12 @@ def open_dataset( if engine is None: engine = _autodetect_engine(filename_or_obj) - backend = _get_backend_cls(engine, engines=plugins.detect_engines()) - open_backend_dataset = backend.open_dataset - open_backend_dataset_parameters = backend.open_dataset_parameters + engines = plugins.list_engines() + backend = _get_backend_cls(engine, engines=engines) decoders = resolve_decoders_kwargs( decode_cf, - open_backend_dataset_parameters=open_backend_dataset_parameters, + open_backend_dataset_parameters=backend.open_dataset_parameters, mask_and_scale=mask_and_scale, decode_times=decode_times, decode_timedelta=decode_timedelta, @@ -236,7 +236,7 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - backend_ds = open_backend_dataset( + backend_ds = backend.open_dataset( filename_or_obj, drop_variables=drop_variables, **decoders, diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 63e8a6692de..9e78a3b71e8 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -1,10 +1,9 @@ import inspect import itertools -import typing as T import warnings from functools import lru_cache -import entrypoints # type: ignore +import pkg_resources class BackendEntrypoint: @@ -15,43 +14,28 @@ def __init__(self, open_dataset, open_dataset_parameters=None): self.open_dataset_parameters = open_dataset_parameters -def get_entrypoint_id(entrypoint): - name = entrypoint.name - obj = entrypoint.object_name or "" - module = entrypoint.module_name - return (name, f"{module}:{obj}") - - -def filter_unique_ids(backend_entrypoints): - entrypoints_id = set() - backend_entrypoints_unique = [] - for entrypoint in backend_entrypoints: - id = get_entrypoint_id(entrypoint) - if id not in entrypoints_id: - backend_entrypoints_unique.append(entrypoint) - entrypoints_id.add(id) - return backend_entrypoints_unique - - -def warning_on_entrypoints_conflict(backend_entrypoints, backend_entrypoints_all): +def remove_duplicates(backend_entrypoints): # sort and group entrypoints by name - key_name = lambda ep: ep.name - backend_entrypoints_all_unique_ids = sorted(backend_entrypoints_all, key=key_name) - backend_entrypoints_ids_grouped = itertools.groupby( - backend_entrypoints_all_unique_ids, key=key_name + backend_entrypoints = sorted(backend_entrypoints, key=lambda ep: ep.name) + backend_entrypoints_grouped = itertools.groupby( + backend_entrypoints, key=lambda ep: ep.name ) - # check if there are multiple entrypoints for the same name - for name, matches in backend_entrypoints_ids_grouped: + unique_backend_entrypoints = [] + for name, matches in backend_entrypoints_grouped: matches = list(matches) + unique_backend_entrypoints.append(matches[0]) matches_len = len(matches) if matches_len > 1: - selected_entrypoint = backend_entrypoints[name] + selected_module_name = matches[0].module_name + all_module_names = [e.module_name for e in matches] warnings.warn( f"\nFound {matches_len} entrypoints for the engine name {name}:" - f"\n {matches}.\n It will be used: {selected_entrypoint}.", + f"\n {all_module_names}.\n It will be used: {selected_module_name}.", + RuntimeWarning, ) + return unique_backend_entrypoints def detect_parameters(open_dataset): @@ -66,24 +50,29 @@ def detect_parameters(open_dataset): f"All the parameters in {open_dataset!r} signature should be explicit. " "*args and **kwargs is not supported" ) - return set(parameters) - + return tuple(parameters) -@lru_cache(maxsize=1) -def detect_engines(): - backend_entrypoints = entrypoints.get_group_named("xarray.backends") - backend_entrypoints_all = entrypoints.get_group_all("xarray.backends") - backend_entrypoints_all = filter_unique_ids(backend_entrypoints_all) - if len(backend_entrypoints_all) != len(backend_entrypoints): - warning_on_entrypoints_conflict(backend_entrypoints, backend_entrypoints_all) - - engines: T.Dict[str, T.Dict[str, T.Any]] = {} - for name, backend in backend_entrypoints.items(): - backend = backend.load() +def create_engines_dict(backend_entrypoints): + engines = {} + for backend_ep in backend_entrypoints: + name = backend_ep.name + backend = backend_ep.load() engines[name] = backend + return engines + +def set_missing_parameters(engines): + for name, backend in engines.items(): if backend.open_dataset_parameters is None: open_dataset = backend.open_dataset backend.open_dataset_parameters = detect_parameters(open_dataset) + + +@lru_cache(maxsize=1) +def list_engines(): + entrypoints = pkg_resources.iter_entry_points("xarray.backends") + backend_entrypoints = remove_duplicates(entrypoints) + engines = create_engines_dict(backend_entrypoints) + set_missing_parameters(engines) return engines diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py new file mode 100644 index 00000000000..7cedc2db254 --- /dev/null +++ b/xarray/tests/test_plugins.py @@ -0,0 +1,97 @@ +from unittest import mock + +import pkg_resources +import pytest + +from xarray.backends import plugins + + +def dummy_open_dataset_args(filename_or_obj, *args): + pass + + +def dummy_open_dataset_kwargs(filename_or_obj, **kwargs): + pass + + +def dummy_open_dataset(filename_or_obj, *, decoder): + pass + + +@pytest.fixture +def dummy_duplicated_entrypoints(): + specs = [ + "engine1 = xarray.tests.test_plugins:backend_1", + "engine1 = xarray.tests.test_plugins:backend_2", + "engine2 = xarray.tests.test_plugins:backend_1", + "engine2 = xarray.tests.test_plugins:backend_2", + ] + eps = [] + for spec in specs: + eps.append(pkg_resources.EntryPoint.parse(spec)) + return eps + + +def test_remove_duplicates(dummy_duplicated_entrypoints): + entrypoints = plugins.remove_duplicates(dummy_duplicated_entrypoints) + assert len(entrypoints) == 2 + + +def test_remove_duplicates_wargnings(dummy_duplicated_entrypoints): + + with pytest.warns(RuntimeWarning) as record: + _ = plugins.remove_duplicates(dummy_duplicated_entrypoints) + + assert len(record) == 2 + message0 = str(record[0].message) + message1 = str(record[1].message) + assert "entrypoints" in message0 + assert "entrypoints" in message1 + + +@mock.patch("pkg_resources.EntryPoint.load", mock.MagicMock(return_value=None)) +def test_create_engines_dict(): + specs = [ + "engine1 = xarray.tests.test_plugins:backend_1", + "engine2 = xarray.tests.test_plugins:backend_2", + ] + entrypoints = [] + for spec in specs: + entrypoints.append(pkg_resources.EntryPoint.parse(spec)) + engines = plugins.create_engines_dict(entrypoints) + assert len(engines) == 2 + assert engines.keys() == set(("engine1", "engine2")) + + +def test_set_missing_parameters(): + backend_1 = plugins.BackendEntrypoint(dummy_open_dataset) + backend_2 = plugins.BackendEntrypoint(dummy_open_dataset, ("filename_or_obj",)) + engines = {"engine_1": backend_1, "engine_2": backend_2} + plugins.set_missing_parameters(engines) + + assert len(engines) == 2 + engine_1 = engines["engine_1"] + assert engine_1.open_dataset_parameters == ("filename_or_obj", "decoder") + engine_2 = engines["engine_2"] + assert engine_2.open_dataset_parameters == ("filename_or_obj",) + + +def test_set_missing_parameters_raise_error(): + + backend = plugins.BackendEntrypoint(dummy_open_dataset_args) + with pytest.raises(TypeError): + plugins.set_missing_parameters({"engine": backend}) + + backend = plugins.BackendEntrypoint( + dummy_open_dataset_args, ("filename_or_obj", "decoder") + ) + plugins.set_missing_parameters({"engine": backend}) + + backend = plugins.BackendEntrypoint(dummy_open_dataset_kwargs) + with pytest.raises(TypeError): + plugins.set_missing_parameters({"engine": backend}) + + backend = plugins.BackendEntrypoint( + dummy_open_dataset_kwargs, ("filename_or_obj", "decoder") + ) + plugins.set_missing_parameters({"engine": backend}) From 9a14597c7d97006267a869d06bf423fd6ea0881a Mon Sep 17 00:00:00 2001 From: aurghs <35919497+aurghs@users.noreply.github.com> Date: Thu, 3 Dec 2020 08:24:34 +0100 Subject: [PATCH 14/21] fix typo Co-authored-by: keewis --- xarray/tests/test_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 7cedc2db254..b7cb97fbfb2 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -37,7 +37,7 @@ def test_remove_duplicates(dummy_duplicated_entrypoints): assert len(entrypoints) == 2 -def test_remove_duplicates_wargnings(dummy_duplicated_entrypoints): +def test_remove_duplicates_warnings(dummy_duplicated_entrypoints): with pytest.warns(RuntimeWarning) as record: _ = plugins.remove_duplicates(dummy_duplicated_entrypoints) From 57e5e21c353aa2ed5a08d0e25a9fa6ccd0de1ce4 Mon Sep 17 00:00:00 2001 From: aurghs <35919497+aurghs@users.noreply.github.com> Date: Thu, 3 Dec 2020 08:24:57 +0100 Subject: [PATCH 15/21] style Co-authored-by: keewis --- xarray/tests/test_plugins.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index b7cb97fbfb2..c3bfbc9afbe 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -26,9 +26,10 @@ def dummy_duplicated_entrypoints(): "engine2 = xarray.tests.test_plugins:backend_1", "engine2 = xarray.tests.test_plugins:backend_2", ] - eps = [] - for spec in specs: - eps.append(pkg_resources.EntryPoint.parse(spec)) + eps = [ + pkg_resources.EntryPoint.parse(spec) + for spec in specs + ] return eps From a302c87312c660e822c6371920c49a473b7919d7 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 3 Dec 2020 09:14:19 +0100 Subject: [PATCH 16/21] style --- xarray/tests/test_plugins.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index c3bfbc9afbe..fa02f451a71 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -56,9 +56,7 @@ def test_create_engines_dict(): "engine1 = xarray.tests.test_plugins:backend_1", "engine2 = xarray.tests.test_plugins:backend_2", ] - entrypoints = [] - for spec in specs: - entrypoints.append(pkg_resources.EntryPoint.parse(spec)) + entrypoints = [pkg_resources.EntryPoint.parse(spec) for spec in specs] engines = plugins.create_engines_dict(entrypoints) assert len(engines) == 2 assert engines.keys() == set(("engine1", "engine2")) From f0821c2b1ca8189f4964f6f15909c9f06d612766 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Wed, 9 Dec 2020 14:18:11 +0100 Subject: [PATCH 17/21] Code style --- xarray/backends/apiv2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index a1f48e74606..c3f160cbc18 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -55,10 +55,7 @@ def _chunk_ds( for k, v in backend_ds.variables.items(): var_chunks = _get_chunk(k, v, chunks) variables[k] = _maybe_chunk( - k, - v, - var_chunks, - overwrite_encoded_chunks=overwrite_encoded_chunks, + k, v, var_chunks, overwrite_encoded_chunks=overwrite_encoded_chunks, ) ds = backend_ds._replace(variables) return ds From f73f062d8a6154b46ed999e1e01eba2a06e688c7 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Wed, 9 Dec 2020 14:26:32 +0100 Subject: [PATCH 18/21] Code style --- xarray/backends/apiv2.py | 5 ++++- xarray/tests/test_plugins.py | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index c3f160cbc18..a1f48e74606 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -55,7 +55,10 @@ def _chunk_ds( for k, v in backend_ds.variables.items(): var_chunks = _get_chunk(k, v, chunks) variables[k] = _maybe_chunk( - k, v, var_chunks, overwrite_encoded_chunks=overwrite_encoded_chunks, + k, + v, + var_chunks, + overwrite_encoded_chunks=overwrite_encoded_chunks, ) ds = backend_ds._replace(variables) return ds diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index fa02f451a71..7e9bb58f140 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -26,10 +26,7 @@ def dummy_duplicated_entrypoints(): "engine2 = xarray.tests.test_plugins:backend_1", "engine2 = xarray.tests.test_plugins:backend_2", ] - eps = [ - pkg_resources.EntryPoint.parse(spec) - for spec in specs - ] + eps = [pkg_resources.EntryPoint.parse(spec) for spec in specs] return eps From 78b18663961c2506cd76542db83894457c899ed8 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 9 Dec 2020 23:41:37 +0100 Subject: [PATCH 19/21] fix: updated plugins.ENGINES with plugins.list_engines() --- xarray/backends/apiv2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index a1f48e74606..d4c7c8cd38d 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -267,7 +267,7 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - open_backend_dataset = _get_backend_cls(engine, engines=plugins.ENGINES)[ + open_backend_dataset = _get_backend_cls(engine, engines=plugins.list_engines())[ "open_dataset" ] backend_ds = open_backend_dataset( From ad023c48f94476fb4e600066d050d06e553e12e4 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 10 Dec 2020 08:50:01 +0100 Subject: [PATCH 20/21] fix --- xarray/backends/apiv2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index d4c7c8cd38d..6b0990a5194 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -266,10 +266,9 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - - open_backend_dataset = _get_backend_cls(engine, engines=plugins.list_engines())[ - "open_dataset" - ] + open_backend_dataset = _get_backend_cls( + engine, engines=plugins.list_engines() + ).open_dataset backend_ds = open_backend_dataset( filename_or_obj, drop_variables=drop_variables, From 14bf314e7b670fdd07e089a61c257c488ea540a3 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 10 Dec 2020 10:09:47 +0100 Subject: [PATCH 21/21] One more correctness fix of the latest merge from master --- xarray/backends/apiv2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/xarray/backends/apiv2.py b/xarray/backends/apiv2.py index 6b0990a5194..b3f5343df7c 100644 --- a/xarray/backends/apiv2.py +++ b/xarray/backends/apiv2.py @@ -266,10 +266,7 @@ def open_dataset( backend_kwargs = backend_kwargs.copy() overwrite_encoded_chunks = backend_kwargs.pop("overwrite_encoded_chunks", None) - open_backend_dataset = _get_backend_cls( - engine, engines=plugins.list_engines() - ).open_dataset - backend_ds = open_backend_dataset( + backend_ds = backend.open_dataset( filename_or_obj, drop_variables=drop_variables, **decoders,