From bb88302a431fa86e782241ece57625eeb9636bfc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 01:09:26 +0000 Subject: [PATCH 1/4] ci(release): update to development version 0.0.9 --- README.md | 2 +- docs/conf.py | 2 +- modflow_devtools/__init__.py | 2 +- version.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b47c89ca..a5407018 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MODFLOW developer tools -### Version 0.0.8 — release candidate +### Version 0.0.9 — release candidate [![GitHub tag](https://img.shields.io/github/tag/MODFLOW-USGS/modflow-devtools.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/tags/latest) [![PyPI Version](https://img.shields.io/pypi/v/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) [![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) diff --git a/docs/conf.py b/docs/conf.py index e7882897..a749c93a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = 'modflow-devtools' author = 'MODFLOW Team' -release = 0.0.8 +release = 0.0.9 # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index 415edb28..5a2d6a6c 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -1,6 +1,6 @@ __author__ = "Joseph D. Hughes" __date__ = "Dec 29, 2022" -__version__ = "0.0.8" +__version__ = "0.0.9" __maintainer__ = "Joseph D. Hughes" __email__ = "jdhughes@usgs.gov" __status__ = "Production" diff --git a/version.txt b/version.txt index 7d6b3eb3..429d94ae 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.8 \ No newline at end of file +0.0.9 \ No newline at end of file From 23593df7fb427d6de1d33f9aa408697d2536e473 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Dec 2022 14:17:43 -0500 Subject: [PATCH 2/4] refactor(fixtures): fix/refactor model-loading fixtures (#33) * refactor test model fixtures as path to namefile, not model dir * refactor model-loading utility functions for distinct use cases * expand tests * update docs --- docs/md/fixtures.md | 26 +++-- modflow_devtools/fixtures.py | 20 ++-- modflow_devtools/misc.py | 65 +++++++---- modflow_devtools/test/test_fixtures.py | 12 ++- modflow_devtools/test/test_misc.py | 144 ++++++++++++++++++++----- 5 files changed, 198 insertions(+), 69 deletions(-) diff --git a/docs/md/fixtures.md b/docs/md/fixtures.md index 961725ac..a16cd5e7 100644 --- a/docs/md/fixtures.md +++ b/docs/md/fixtures.md @@ -2,7 +2,7 @@ Several `pytest` fixtures are provided to help with testing. -## Keepable temporary directory fixtures +## Keepable temporary directories Tests often need to exercise code that reads from and/or writes to disk. The test harness may also need to create test data during setup and clean up the filesystem on teardown. Temporary directories are built into `pytest` via the [`tmp_path`](https://docs.pytest.org/en/latest/how-to/tmp_path.html#the-tmp-path-fixture) and `tmp_path_factory` fixtures. @@ -40,9 +40,9 @@ pytest --keep There is also a `--keep-failed ` option which preserves outputs only from failing test cases. -## Model-loading fixtures +## Loading example models -Fixtures are provided to load models from the MODFLOW 6 example and test model repositories and feed them to test functions. Models can be loaded from: +Fixtures are provided to find and enumerate models from the MODFLOW 6 example and test model repositories and feed them to test functions. Models can be loaded from: - [`MODFLOW-USGS/modflow6-examples`](https://github.com/MODFLOW-USGS/modflow6-examples) - [`MODFLOW-USGS/modflow6-testmodels`](https://github.com/MODFLOW-USGS/modflow6-testmodels) @@ -61,14 +61,15 @@ To use these fixtures, the environment variable `REPOS_PATH` must point to the l ### Test models -The `test_model_mf5to6`, `test_model_mf6` and `large_test_model` fixtures are each a `Path` to the directory containing the model's namefile. For instance, to load `mf5to6` models from the [`MODFLOW-USGS/modflow6-testmodels`](https://github.com/MODFLOW-USGS/modflow6-testmodels) repository: +The `test_model_mf5to6`, `test_model_mf6` and `large_test_model` fixtures are each a `Path` to the model's namefile. For example:, to load `mf5to6` models from the repository: ```python -def test_mf5to6_model(testmodel_mf5to6): - assert testmodel_mf5to6.is_dir() +def test_mf5to6_model(test_model_mf5to6): + assert test_model_mf5to6.is_file() + assert test_model_mf5to6.suffix == ".nam" ``` -This test function will be parametrized with all `mf5to6` models found in the `testmodels` repository (likewise for `mf6` models, and for large test models in their own repository). +This test function will be parametrized with all `mf5to6` models found in the [`MODFLOW-USGS/modflow6-testmodels`](https://github.com/MODFLOW-USGS/modflow6-testmodels). ### Example scenarios @@ -87,4 +88,13 @@ def test_example_scenario(tmp_path, example_scenario): model_ws.mkdir() # load and run model # ... -``` \ No newline at end of file +``` + +### Utility functions + +Model-loading fixtures use a set of utility functions to find and enumerate models. These functions can be imported from `modflow_devtools.misc` for use in other contexts: + +- `get_model_dir_paths()` +- `get_namefile_paths()` + +See this project's test suite for usage examples. diff --git a/modflow_devtools/fixtures.py b/modflow_devtools/fixtures.py index 5efdccca..8f34a03d 100644 --- a/modflow_devtools/fixtures.py +++ b/modflow_devtools/fixtures.py @@ -6,7 +6,11 @@ from typing import Dict, List, Optional import pytest -from modflow_devtools.misc import get_model_paths, get_packages +from modflow_devtools.misc import ( + get_model_dir_paths, + get_namefile_paths, + get_packages, +) # temporary directory fixtures @@ -174,7 +178,7 @@ def pytest_generate_tests(metafunc): key = "test_model_mf6" if key in metafunc.fixturenames: models = ( - get_model_paths( + get_namefile_paths( Path(repos_path) / "modflow6-testmodels" / "mf6", prefix="test", excluded=["test205_gwtbuy-henrytidal"], @@ -184,12 +188,12 @@ def pytest_generate_tests(metafunc): if repos_path else [] ) - metafunc.parametrize(key, models, ids=[m.name for m in models]) + metafunc.parametrize(key, models, ids=[str(m) for m in models]) key = "test_model_mf5to6" if key in metafunc.fixturenames: models = ( - get_model_paths( + get_namefile_paths( Path(repos_path) / "modflow6-testmodels" / "mf5to6", prefix="test", namefile="*.nam", @@ -200,15 +204,15 @@ def pytest_generate_tests(metafunc): if repos_path else [] ) - metafunc.parametrize(key, models, ids=[m.name for m in models]) + metafunc.parametrize(key, models, ids=[str(m) for m in models]) key = "large_test_model" if key in metafunc.fixturenames: models = ( - get_model_paths( + get_namefile_paths( Path(repos_path) / "modflow6-largetestmodels", prefix="test", - namefile="*.nam", + namefile="mfsim.nam", excluded=[], selected=models_selected, packages=packages_selected, @@ -216,7 +220,7 @@ def pytest_generate_tests(metafunc): if repos_path else [] ) - metafunc.parametrize(key, models, ids=[m.name for m in models]) + metafunc.parametrize(key, models, ids=[str(m) for m in models]) key = "example_scenario" if key in metafunc.fixturenames: diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index 6731797c..8c0738c5 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -160,29 +160,27 @@ def has_package(namefile_path: PathLike, package: str) -> bool: return package.lower in packages -def get_model_paths( +def get_namefile_paths( path: PathLike, prefix: str = None, namefile: str = "mfsim.nam", excluded=None, selected=None, packages=None, -) -> List[Path]: +): """ - Find models recursively in the given location. - Models can be filtered or excluded by pattern, - filtered by packages used or naming convention - for namefiles, or by parent folder name prefix. - The path to the model folder (i.e., the folder - containing the model's namefile) is returned. + Find namefiles recursively in the given location. + Namefiles can be filtered or excluded by pattern, + by parent directory name prefix or pattern, or by + packages used. """ # if path doesn't exist, return empty list if not Path(path).is_dir(): return [] - # find namfiles - namfile_paths = [ + # find namefiles + paths = [ p for p in Path(path).rglob( f"{prefix}*/**/{namefile}" if prefix else namefile @@ -190,36 +188,59 @@ def get_model_paths( ] # remove excluded - namfile_paths = [ + paths = [ p - for p in namfile_paths + for p in paths if (not excluded or not any(e in str(p) for e in excluded)) ] # filter by package if packages: filtered = [] - for nfp in namfile_paths: + for nfp in paths: nf_pkgs = get_packages(nfp) shared = set(nf_pkgs).intersection( set([p.lower() for p in packages]) ) if any(shared): filtered.append(nfp) - namfile_paths = filtered - - # get model folder paths - model_paths = [p.parent for p in namfile_paths] + paths = filtered # filter by model name if selected: - model_paths = [ - model - for model in model_paths - if any(s in model.name for s in selected) + paths = [ + namfile_path + for (namfile_path, model_path) in zip( + paths, [p.parent for p in paths] + ) + if any(s in model_path.name for s in selected) ] - return sorted(model_paths) + return sorted(paths) + + +def get_model_dir_paths( + path: PathLike, + prefix: str = None, + namefile: str = "mfsim.nam", + excluded=None, + selected=None, + packages=None, +) -> List[Path]: + """ + Find model directories recursively in the given location. + A model directory is any directory containing one or more + namefiles. Model directories can be filtered or excluded, + by prefix, pattern, namefile name, or packages used. + """ + + namefile_paths = get_namefile_paths( + path, prefix, namefile, excluded, selected, packages + ) + model_paths = sorted( + list(set([p.parent for p in namefile_paths if p.parent.name])) + ) + return model_paths def is_connected(hostname): diff --git a/modflow_devtools/test/test_fixtures.py b/modflow_devtools/test/test_fixtures.py index 2a06e52c..3d9672d4 100644 --- a/modflow_devtools/test/test_fixtures.py +++ b/modflow_devtools/test/test_fixtures.py @@ -249,16 +249,18 @@ def test_example_scenario(example_scenario): def test_test_model_mf6(test_model_mf6): assert isinstance(test_model_mf6, Path) - assert (test_model_mf6 / "mfsim.nam").is_file() + assert test_model_mf6.is_file() + assert test_model_mf6.name == "mfsim.nam" def test_test_model_mf5to6(test_model_mf5to6): assert isinstance(test_model_mf5to6, Path) - assert any(list(test_model_mf5to6.glob("*.nam"))) + assert test_model_mf5to6.is_file() + assert test_model_mf5to6.suffix == ".nam" def test_large_test_model(large_test_model): + print(large_test_model) assert isinstance(large_test_model, Path) - assert (large_test_model / "mfsim.nam").is_file() or any( - list(large_test_model.glob("*.nam")) - ) + assert large_test_model.is_file() + assert large_test_model.suffix == ".nam" diff --git a/modflow_devtools/test/test_misc.py b/modflow_devtools/test/test_misc.py index 20e580eb..5a88c79b 100644 --- a/modflow_devtools/test/test_misc.py +++ b/modflow_devtools/test/test_misc.py @@ -4,7 +4,12 @@ from typing import List import pytest -from modflow_devtools.misc import get_model_paths, get_packages, set_dir +from modflow_devtools.misc import ( + get_model_dir_paths, + get_namefile_paths, + get_packages, + set_dir, +) def test_set_dir(tmp_path): @@ -15,6 +20,8 @@ def test_set_dir(tmp_path): _repos_path = Path(environ.get("REPOS_PATH")).expanduser().absolute() +_largetestmodels_repo_path = _repos_path / "modflow6-largetestmodels" +_largetestmodel_paths = sorted(list(_largetestmodels_repo_path.glob("test*"))) _examples_repo_path = _repos_path / "modflow6-examples" _examples_path = _examples_repo_path / "examples" _example_paths = ( @@ -31,45 +38,130 @@ def test_has_packages(): assert set(packages) == {"tdis", "gwf", "ims"} -@pytest.mark.skipif(not any(_example_paths), reason="examples not found") -def test_get_model_paths(): - def get_expected(path, pattern="mfsim.nam") -> List[Path]: - folders = [] - for root, dirs, files in os.walk(path): - for d in dirs: - p = Path(root) / d - if any(p.glob(pattern)): - folders.append(p) - return sorted(folders) - - expected_paths = get_expected(_examples_path) - paths = get_model_paths(_examples_path) +def get_expected_model_dirs(path, pattern="mfsim.nam") -> List[Path]: + folders = [] + for root, dirs, files in os.walk(path): + for d in dirs: + p = Path(root) / d + if any(p.glob(pattern)): + folders.append(p) + return sorted(list(set(folders))) + + +def get_expected_namefiles(path, pattern="mfsim.nam") -> List[Path]: + folders = [] + for root, dirs, files in os.walk(path): + for d in dirs: + p = Path(root) / d + found = list(p.glob(pattern)) + folders = folders + found + return sorted(list(set(folders))) + + +@pytest.mark.skipif( + not any(_example_paths), reason="modflow6-examples repo not found" +) +def test_get_model_dir_paths_examples(): + expected_paths = get_expected_model_dirs(_examples_path) + paths = get_model_dir_paths(_examples_path) + assert paths == sorted(list(set(paths))) # no duplicates assert set(expected_paths) == set(paths) - expected_paths = get_expected(_examples_path, "*.nam") - paths = get_model_paths(_examples_path, namefile="*.nam") + expected_paths = get_expected_model_dirs(_examples_path, "*.nam") + paths = get_model_dir_paths(_examples_path, namefile="*.nam") + assert paths == sorted(list(set(paths))) assert set(expected_paths) == set(paths) -@pytest.mark.skipif(not any(_example_paths), reason="examples not found") -def test_get_model_paths_exclude_patterns(): - paths = get_model_paths(_examples_path, excluded=["gwt"]) - assert len(paths) == 63 +@pytest.mark.skipif( + not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found" +) +def test_get_model_dir_paths_largetestmodels(): + expected_paths = get_expected_model_dirs(_examples_path) + paths = get_model_dir_paths(_examples_path) + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + expected_paths = get_expected_model_dirs(_examples_path) + paths = get_model_dir_paths(_examples_path) + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + +@pytest.mark.skipif( + not any(_largetestmodel_paths) or not any(_example_paths), + reason="repos not found", +) +@pytest.mark.parametrize( + "models", [(_examples_path, 63), (_largetestmodels_repo_path, 15)] +) +def test_get_model_dir_paths_exclude_patterns(models): + path, expected_count = models + paths = get_model_dir_paths(path, excluded=["gwt"]) + assert len(paths) == expected_count + + +@pytest.mark.skipif( + not any(_example_paths), reason="modflow6-examples repo not found" +) +def test_get_namefile_paths_examples(): + expected_paths = get_expected_namefiles(_examples_path) + paths = get_namefile_paths(_examples_path) + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + expected_paths = get_expected_namefiles(_examples_path, "*.nam") + paths = get_namefile_paths(_examples_path, namefile="*.nam") + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + +@pytest.mark.skipif( + not any(_largetestmodel_paths), reason="modflow6-largetestmodels not found" +) +def test_get_namefile_paths_largetestmodels(): + expected_paths = get_expected_namefiles(_largetestmodels_repo_path) + paths = get_namefile_paths(_largetestmodels_repo_path) + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + expected_paths = get_expected_namefiles(_largetestmodels_repo_path) + paths = get_namefile_paths(_largetestmodels_repo_path) + assert paths == sorted(list(set(paths))) + assert set(expected_paths) == set(paths) + + +@pytest.mark.skipif( + not any(_largetestmodel_paths) or not any(_example_paths), + reason="repos not found", +) +@pytest.mark.parametrize( + "models", [(_examples_path, 43), (_largetestmodels_repo_path, 18)] +) +def test_get_namefile_paths_exclude_patterns(models): + path, expected_count = models + paths = get_namefile_paths(path, excluded=["gwf"]) + assert len(paths) == expected_count @pytest.mark.skipif(not any(_example_paths), reason="examples not found") -def test_get_model_paths_select_prefix(): - paths = get_model_paths(_examples_path, prefix="ex2") +def test_get_namefile_paths_select_prefix(): + paths = get_namefile_paths(_examples_path, prefix="ex2") assert not any(paths) + paths = get_namefile_paths(_examples_path, prefix="ex") + assert any(paths) + @pytest.mark.skipif(not any(_example_paths), reason="examples not found") -def test_get_model_paths_select_patterns(): - paths = get_model_paths(_examples_path, selected=["gwf"]) +def test_get_namefile_paths_select_patterns(): + paths = get_namefile_paths(_examples_path, selected=["gwf"]) assert len(paths) == 70 @pytest.mark.skipif(not any(_example_paths), reason="examples not found") -def test_get_model_paths_select_packages(): - paths = get_model_paths(_examples_path, namefile="*.nam", packages=["wel"]) +def test_get_namefile_paths_select_packages(): + paths = get_namefile_paths( + _examples_path, namefile="*.nam", packages=["wel"] + ) assert len(paths) == 64 From 126d52366492eb5eea4e40755321c6e67770857e Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Dec 2022 14:26:15 -0500 Subject: [PATCH 3/4] docs(package): fix package development status (#35) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 432cd066..4cbf8890 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ license_files = LICENSE.md platform = Windows, Mac OS-X, Linux keywords = MODFLOW, development classifiers = - Development Status :: 5 - Production/Stable + Development Status :: 3 - Alpha Intended Audience :: Science/Research License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication Operating System :: Microsoft :: Windows From 1d447ddbaa1b8eb45d32f271f88987613022721d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 19:29:31 +0000 Subject: [PATCH 4/4] ci(release): set version to 0.1.0, update changelog --- HISTORY.md | 6 ++++++ README.md | 2 +- docs/conf.py | 2 +- modflow_devtools/__init__.py | 2 +- version.txt | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 07c97932..21901db5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,9 @@ +### Version 0.1.0 + +#### Refactoring + +* [refactor(fixtures)](https://github.com/modflowpy/flopy/commit/23593df7fb427d6de1d33f9aa408697d2536e473): Fix/refactor model-loading fixtures (#33). Committed by w-bonelli on 2022-12-29. + ### Version 0.0.8 #### Bug fixes diff --git a/README.md b/README.md index a5407018..29660ba2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MODFLOW developer tools -### Version 0.0.9 — release candidate +### Version 0.1.0 — release candidate [![GitHub tag](https://img.shields.io/github/tag/MODFLOW-USGS/modflow-devtools.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/tags/latest) [![PyPI Version](https://img.shields.io/pypi/v/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) [![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) diff --git a/docs/conf.py b/docs/conf.py index a749c93a..5c3aa8a5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = 'modflow-devtools' author = 'MODFLOW Team' -release = 0.0.9 +release = 0.1.0 # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index 5a2d6a6c..3a19236d 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -1,6 +1,6 @@ __author__ = "Joseph D. Hughes" __date__ = "Dec 29, 2022" -__version__ = "0.0.9" +__version__ = "0.1.0" __maintainer__ = "Joseph D. Hughes" __email__ = "jdhughes@usgs.gov" __status__ = "Production" diff --git a/version.txt b/version.txt index 429d94ae..6c6aa7cb 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.0.9 \ No newline at end of file +0.1.0 \ No newline at end of file