From d087dfeaec9b103cb7a32eafaf4ba924d9c61635 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 9 Apr 2025 17:05:46 +0200 Subject: [PATCH 1/7] Add parcels._datasets namespace --- parcels/_datasets/__init__.py | 46 ++++++++++++++++++++++ parcels/_datasets/structured/__init__.py | 1 + parcels/_datasets/unstructured/__init__.py | 1 + 3 files changed, 48 insertions(+) create mode 100644 parcels/_datasets/__init__.py create mode 100644 parcels/_datasets/structured/__init__.py create mode 100644 parcels/_datasets/unstructured/__init__.py diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py new file mode 100644 index 0000000000..c65685c020 --- /dev/null +++ b/parcels/_datasets/__init__.py @@ -0,0 +1,46 @@ +""" +Datasets compatible with Parcels. + +This subpackage uses xarray to generate *idealised* structured and unstructured hydrodynamical datasets that are compatible with Parcels. The goals are three-fold: + +1. To provide users with documentation for the types of datasets they can expect Parcels to work with. +2. To supply our tutorials with hydrodynamical datasets. +3. To offer developers datasets for use in test cases. + +Note that this subpackage is part of the private API for Parcels. Users should not rely directly on the functions defined within this module. Instead, if you want to generate your own datasets, copy the functions from this module into your own code. + +Developers, note that only idealised datasets that are (a) quick to generate, and (b) only use dependencies already shipped with Parcels, should be added to this subpackage. Real world datasets should be added to the `parcels-data` repository. No data files should be added to this subpackage. + +Parcels Dataset Philosophy +------------------------- + +When adding example datasets, there may be a tension between wanting to add a specific example or wanting to add machinery to generate completely arbitrary datasets (e.g., with different grid resolutions, with different ranges, with different datetimes etc.). There are trade-offs to both approaches: + +Working with specific hardcoded examples: + +* Pros + * the example is stable and self-contained + * easy to see exactly what the dataset is, there little to no dependency on other functions defined in the same module + * datasets don't "break" due to changes in other functions (e.g., grid edges becoming out of sync with grid centres) +* Cons + * inflexible for use in tests where you want to test a large range of datasets, or you want to test a specific resolution + +Working with generated datasets is the opposite of all the above. + +Most of the time we only want a single dataset. For example, for use in a tutorial, or for testing a specific feature of Parcels - such as (in the case of structured grids) checking that the grid from a certain vendor is correctly parsed, or checking that indexing is correctly picked up. As such, one should often opt for hardcoded datasets. These are more stable and easier to see exactly what the dataset is. We may have specific examples that become the default "go to" dataset for testing when we don't care about the specific details of the dataset. + +Sometimes we may want to test Parcels against a whole range of datasets varying in a certain way - to ensure Parcels works as expected for subspace of possible datasets. For these, we should add machinery to create generated datasets. + +Structure +-------- + +This subpackage is broken down into structured and unstructured parts. Each of these have common submodules: + +* ``providers`` -> hardcoded datasets with the intention of mimicking datasets from a certain provider +* ``generic`` -> hardcoded datasets that are generic, and not tied to a certain provider (instead focusing on the fundamental properties of the dataset) +* ``generated`` -> functions to generate datasets with varying properties +* ``utils`` -> any utility functions necessary related to either generating or validating datasets + +There may be extra submodules than the ones listed above. + +""" diff --git a/parcels/_datasets/structured/__init__.py b/parcels/_datasets/structured/__init__.py new file mode 100644 index 0000000000..6d040b617a --- /dev/null +++ b/parcels/_datasets/structured/__init__.py @@ -0,0 +1 @@ +"""Structured datasets.""" diff --git a/parcels/_datasets/unstructured/__init__.py b/parcels/_datasets/unstructured/__init__.py new file mode 100644 index 0000000000..2fa5f48f04 --- /dev/null +++ b/parcels/_datasets/unstructured/__init__.py @@ -0,0 +1 @@ +"""Unstructured datasets.""" From f1df5b5527a655bf3174409a085319ffb7975300 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 9 Apr 2025 17:06:10 +0200 Subject: [PATCH 2/7] Add utilities for comparing datasets --- parcels/_datasets/utils.py | 69 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 parcels/_datasets/utils.py diff --git a/parcels/_datasets/utils.py b/parcels/_datasets/utils.py new file mode 100644 index 0000000000..d64c628c46 --- /dev/null +++ b/parcels/_datasets/utils.py @@ -0,0 +1,69 @@ +from typing import Any + +import numpy as np +import xarray as xr + +from parcels._compat import add_note + +_SUPPORTED_ATTR_TYPES = int | float | str | np.ndarray + + +def _print_mismatched_keys(d1: dict[Any, Any], d2: dict[Any, Any]) -> None: + k1 = set(d1.keys()) + k2 = set(d2.keys()) + if len(k1 ^ k2) == 0: + return + print("Mismatched keys:") + print(f"L: {k1 - k2!r}") + print(f"R: {k2 - k1!r}") + + +def assert_common_attrs_equal( + xr_attrs_1: dict[str, _SUPPORTED_ATTR_TYPES], xr_attrs_2: dict[str, _SUPPORTED_ATTR_TYPES], *, verbose: bool = True +) -> None: + d1, d2 = xr_attrs_1, xr_attrs_2 + + common_keys = set(d1.keys()) & set(d2.keys()) + if verbose: + _print_mismatched_keys(d1, d2) + + for key in common_keys: + try: + if isinstance(d1[key], np.ndarray): + np.testing.assert_array_equal(d1[key], d2[key]) + else: + assert d1[key] == d2[key], f"{d1[key]} != {d2[key]}" + except AssertionError as e: + add_note(e, f"error on key {key!r}") + raise + + +def assert_common_variables_common_attrs_equal(ds1: xr.Dataset, ds2: xr.Dataset, *, verbose: bool = True) -> None: + if verbose: + print("Checking dataset attrs...") + + assert_common_attrs_equal(ds1.attrs, ds2.attrs, verbose=verbose) + + ds1_vars = set(ds1.variables) + ds2_vars = set(ds2.variables) + + common_variables = ds1_vars & ds2_vars + if len(ds1_vars ^ ds2_vars) > 0 and verbose: + print("Mismatched variables:") + print(f"L: {ds1_vars - ds2_vars}") + print(f"R: {ds2_vars - ds1_vars}") + + for var in common_variables: + if verbose: + print(f"Checking {var!r} attrs") + assert_common_attrs_equal(ds1[var].attrs, ds2[var].attrs, verbose=verbose) + + +def dataset_repr_diff(ds1: xr.Dataset, ds2: xr.Dataset) -> str: + """Return a text diff of two datasets.""" + repr1 = repr(ds1) + repr2 = repr(ds2) + import difflib + + diff = difflib.ndiff(repr1.splitlines(keepends=True), repr2.splitlines(keepends=True)) + return "".join(diff) From ad694282fd331177f489466941d29dc8595ff963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nick=20Hodgskin=20=28=F0=9F=A6=8E=20Vecko=29?= <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 10 Apr 2025 13:12:19 +0200 Subject: [PATCH 3/7] Update parcels/_datasets/__init__.py Co-authored-by: Erik van Sebille --- parcels/_datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py index c65685c020..3433dd529c 100644 --- a/parcels/_datasets/__init__.py +++ b/parcels/_datasets/__init__.py @@ -9,7 +9,7 @@ Note that this subpackage is part of the private API for Parcels. Users should not rely directly on the functions defined within this module. Instead, if you want to generate your own datasets, copy the functions from this module into your own code. -Developers, note that only idealised datasets that are (a) quick to generate, and (b) only use dependencies already shipped with Parcels, should be added to this subpackage. Real world datasets should be added to the `parcels-data` repository. No data files should be added to this subpackage. +Developers, note that you should only add functions that create idealised datasets to this subpackage if they are (a) quick to generate, and (b) only use dependencies already shipped with Parcels. No data files should be added to this subpackage. Real world data files should be added to the `OceanParcels/parcels-data` repository on GitHub. Parcels Dataset Philosophy ------------------------- From 6d5eb3bded974d510f1798db17de85ec14468474 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:13:53 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- parcels/_datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py index 3433dd529c..0a956bb885 100644 --- a/parcels/_datasets/__init__.py +++ b/parcels/_datasets/__init__.py @@ -9,7 +9,7 @@ Note that this subpackage is part of the private API for Parcels. Users should not rely directly on the functions defined within this module. Instead, if you want to generate your own datasets, copy the functions from this module into your own code. -Developers, note that you should only add functions that create idealised datasets to this subpackage if they are (a) quick to generate, and (b) only use dependencies already shipped with Parcels. No data files should be added to this subpackage. Real world data files should be added to the `OceanParcels/parcels-data` repository on GitHub. +Developers, note that you should only add functions that create idealised datasets to this subpackage if they are (a) quick to generate, and (b) only use dependencies already shipped with Parcels. No data files should be added to this subpackage. Real world data files should be added to the `OceanParcels/parcels-data` repository on GitHub. Parcels Dataset Philosophy ------------------------- From 4406a55b0a9fe90006c3680bb809d2350c73c390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nick=20Hodgskin=20=28=F0=9F=A6=8E=20Vecko=29?= <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:45:01 +0200 Subject: [PATCH 5/7] Update parcels/_datasets/__init__.py Co-authored-by: Erik van Sebille --- parcels/_datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py index 0a956bb885..fc0752ee36 100644 --- a/parcels/_datasets/__init__.py +++ b/parcels/_datasets/__init__.py @@ -20,7 +20,7 @@ * Pros * the example is stable and self-contained - * easy to see exactly what the dataset is, there little to no dependency on other functions defined in the same module + * easy to see exactly what the dataset is, there is little to no dependency on other functions defined in the same module * datasets don't "break" due to changes in other functions (e.g., grid edges becoming out of sync with grid centres) * Cons * inflexible for use in tests where you want to test a large range of datasets, or you want to test a specific resolution From 4d52884771e411df2c2a800133b572f9bf9ad9b1 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:59:11 +0200 Subject: [PATCH 6/7] review feedback --- parcels/_datasets/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py index fc0752ee36..0718a98511 100644 --- a/parcels/_datasets/__init__.py +++ b/parcels/_datasets/__init__.py @@ -14,9 +14,9 @@ Parcels Dataset Philosophy ------------------------- -When adding example datasets, there may be a tension between wanting to add a specific example or wanting to add machinery to generate completely arbitrary datasets (e.g., with different grid resolutions, with different ranges, with different datetimes etc.). There are trade-offs to both approaches: +When adding datasets, there may be a tension between wanting to add a specific dataset or wanting to add machinery to generate completely parameterised datasets (e.g., with different grid resolutions, with different ranges, with different datetimes etc.). There are trade-offs to both approaches: -Working with specific hardcoded examples: +Working with specific hardcoded datasets: * Pros * the example is stable and self-contained @@ -27,17 +27,17 @@ Working with generated datasets is the opposite of all the above. -Most of the time we only want a single dataset. For example, for use in a tutorial, or for testing a specific feature of Parcels - such as (in the case of structured grids) checking that the grid from a certain vendor is correctly parsed, or checking that indexing is correctly picked up. As such, one should often opt for hardcoded datasets. These are more stable and easier to see exactly what the dataset is. We may have specific examples that become the default "go to" dataset for testing when we don't care about the specific details of the dataset. +Most of the time we only want a single dataset. For example, for use in a tutorial, or for testing a specific feature of Parcels - such as (in the case of structured grids) checking that the grid from a certain (ocean) circulation model is correctly parsed, or checking that indexing is correctly picked up. As such, one should often opt for hardcoded datasets. These are more stable and easier to see exactly what the dataset is. We may have specific examples that become the default "go to" dataset for testing when we don't care about the details of the dataset. -Sometimes we may want to test Parcels against a whole range of datasets varying in a certain way - to ensure Parcels works as expected for subspace of possible datasets. For these, we should add machinery to create generated datasets. +Sometimes we may want to test Parcels against a whole range of datasets varying in a certain way - to ensure Parcels works as expected. For these, we should add machinery to create generated datasets. Structure -------- This subpackage is broken down into structured and unstructured parts. Each of these have common submodules: -* ``providers`` -> hardcoded datasets with the intention of mimicking datasets from a certain provider -* ``generic`` -> hardcoded datasets that are generic, and not tied to a certain provider (instead focusing on the fundamental properties of the dataset) +* ``circulation_model`` -> hardcoded datasets with the intention of mimicking datasets from a certain (ocean) circulation model +* ``general`` -> hardcoded datasets that are general, and not tied to a certain (ocean) circulation model. Instead these focus on the fundamental properties of the dataset * ``generated`` -> functions to generate datasets with varying properties * ``utils`` -> any utility functions necessary related to either generating or validating datasets From 42b0a842868d36f02b349936448304716279dd8b Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:00:39 +0200 Subject: [PATCH 7/7] update wording --- parcels/_datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parcels/_datasets/__init__.py b/parcels/_datasets/__init__.py index 0718a98511..c1270f0017 100644 --- a/parcels/_datasets/__init__.py +++ b/parcels/_datasets/__init__.py @@ -37,7 +37,7 @@ This subpackage is broken down into structured and unstructured parts. Each of these have common submodules: * ``circulation_model`` -> hardcoded datasets with the intention of mimicking datasets from a certain (ocean) circulation model -* ``general`` -> hardcoded datasets that are general, and not tied to a certain (ocean) circulation model. Instead these focus on the fundamental properties of the dataset +* ``generic`` -> hardcoded datasets that are generic, and not tied to a certain (ocean) circulation model. Instead these focus on the fundamental properties of the dataset * ``generated`` -> functions to generate datasets with varying properties * ``utils`` -> any utility functions necessary related to either generating or validating datasets