diff --git a/modflow_devtools/fixtures.py b/modflow_devtools/fixtures.py index ce2fd1be..e0d47a95 100644 --- a/modflow_devtools/fixtures.py +++ b/modflow_devtools/fixtures.py @@ -6,7 +6,7 @@ from typing import Dict, List, Optional import pytest -from modflow_devtools.misc import get_mf6_ftypes, get_models +from modflow_devtools.misc import get_model_paths, get_packages # temporary directory fixtures @@ -174,7 +174,7 @@ def pytest_generate_tests(metafunc): key = "test_model_mf6" if key in metafunc.fixturenames: models = ( - get_models( + get_model_paths( Path(repos_path) / "modflow6-testmodels" / "mf6", prefix="test", excluded=["test205_gwtbuy-henrytidal"], @@ -189,7 +189,7 @@ def pytest_generate_tests(metafunc): key = "test_model_mf5to6" if key in metafunc.fixturenames: models = ( - get_models( + get_model_paths( Path(repos_path) / "modflow6-testmodels" / "mf5to6", prefix="test", namefile="*.nam", @@ -205,7 +205,7 @@ def pytest_generate_tests(metafunc): key = "large_test_model" if key in metafunc.fixturenames: models = ( - get_models( + get_model_paths( Path(repos_path) / "modflow6-largetestmodels", prefix="test", namefile="*.nam", @@ -292,7 +292,7 @@ def get_examples(): for name, namefiles in examples.items(): ftypes = [] for namefile in namefiles: - ftype = get_mf6_ftypes(namefile, packages_selected) + ftype = get_packages(namefile, packages_selected) if ftype not in ftypes: ftypes += ftype if len(ftypes) > 0: diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index 8be1c613..151231ae 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -101,20 +101,17 @@ def get_current_branch() -> str: raise ValueError(f"Could not determine current branch: {stderr}") -def get_mf6_ftypes(namefile_path: PathLike, ftypekeys: List[str]) -> List[str]: +def get_packages(namefile_path: PathLike) -> List[str]: """ - Return a list of FTYPES that are in the name file and in ftypekeys. + Return a list of packages used by the model defined in the given namefile. Parameters ---------- - namefile_path : str - path to a MODFLOW 6 name file - ftypekeys : list - list of desired FTYPEs + namefile_path : PathLike + path to MODFLOW 6 name file Returns ------- - ftypes : list - list of FTYPES that match ftypekeys in namefile + list of package types """ with open(namefile_path, "r") as f: lines = f.readlines() @@ -126,22 +123,29 @@ def get_mf6_ftypes(namefile_path: PathLike, ftypekeys: List[str]) -> List[str]: if len(ll) < 2: continue - if ll[0] in ["#", "!"]: + l = ll[0].lower() + if any(l.startswith(c) for c in ["#", "!", "data", "list"]) or l in [ + "begin", + "end", + "memory_print_option", + ]: continue - for key in ftypekeys: - if key.lower() in ll[0].lower(): - ftypes.append(ll[0]) + # strip "6" from package name + l = l.replace("6", "") - return ftypes + ftypes.append(l.lower()) + return list(set(ftypes)) -def has_packages(namefile_path: PathLike, packages: List[str]) -> bool: - ftypes = [item.upper() for item in get_mf6_ftypes(namefile_path, packages)] - return len(ftypes) > 0 +def has_package(namefile_path: PathLike, package: str) -> bool: + """Determines whether the model with the given namefile contains the selected package""" + packages = get_packages(namefile_path) + return package.lower in packages -def get_models( + +def get_model_paths( path: PathLike, prefix: str = None, namefile: str = "mfsim.nam", @@ -150,7 +154,12 @@ def get_models( packages=None, ) -> List[Path]: """ - Find models in the given filesystem location. + Find models recursively in the given location. + Models can be filtered or excluded by pattern, + filtered by packages used or naming convention + for namefiles, or by parent folder name prefix. + The path to the model folder (i.e., the folder + containing the model's namefile) is returned. """ # if path doesn't exist, return empty list @@ -161,7 +170,7 @@ def get_models( namfile_paths = [ p for p in Path(path).rglob( - f"{prefix}*/{namefile}" if prefix else namefile + f"{prefix}*/**/{namefile}" if prefix else namefile ) ] @@ -172,18 +181,22 @@ def get_models( if (not excluded or not any(e in str(p) for e in excluded)) ] - # filter by package (optional) + # filter by package if packages: - namfile_paths = [ - p - for p in namfile_paths - if (has_packages(p, packages) if packages else True) - ] - - # get model dir paths + filtered = [] + for nfp in namfile_paths: + nf_pkgs = get_packages(nfp) + shared = set(nf_pkgs).intersection( + set([p.lower() for p in packages]) + ) + if any(shared): + filtered.append(nfp) + namfile_paths = filtered + + # get model folder paths model_paths = [p.parent for p in namfile_paths] - # filter by model name (optional) + # filter by model name if selected: model_paths = [ model @@ -191,18 +204,13 @@ def get_models( if any(s in model.name for s in selected) ] - # exclude dev examples on master or release branches - branch = get_current_branch() - if "master" in branch.lower() or "release" in branch.lower(): - model_paths = [ - model for model in model_paths if "_dev" not in model.name.lower() - ] - - return model_paths + return sorted(model_paths) def is_connected(hostname): - """See https://stackoverflow.com/a/20913928/ to test hostname.""" + """ + Tests whether the given URL is accessible. + See https://stackoverflow.com/a/20913928/.""" try: host = socket.gethostbyname(hostname) s = socket.create_connection((host, 80), 2) @@ -214,6 +222,8 @@ def is_connected(hostname): def is_in_ci(): + """Determines whether the current process is running GitHub Actions CI""" + # if running in GitHub Actions CI, "CI" variable always set to true # https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables return bool(environ.get("CI", None)) @@ -222,7 +232,7 @@ def is_in_ci(): def is_github_rate_limited() -> Optional[bool]: """ Determines if a GitHub API rate limit is applied to the current IP. - Note that running this function will consume an API request! + Running this function will consume an API request! Returns ------- diff --git a/modflow_devtools/test/test_executables.py b/modflow_devtools/test/test_executables.py index 86cf0aec..9351b9a7 100644 --- a/modflow_devtools/test/test_executables.py +++ b/modflow_devtools/test/test_executables.py @@ -7,17 +7,12 @@ from modflow_devtools.executables import Executables from modflow_devtools.misc import add_sys_path, get_suffixes -_bin_path = Path(environ.get("BIN_PATH")).expanduser() +_bin_path = Path(environ.get("BIN_PATH")).expanduser().absolute() _ext, _ = get_suffixes(sys.platform) -@pytest.fixture -def bin_path(module_tmpdir) -> Path: - return _bin_path.absolute() - - @pytest.mark.skipif(not _bin_path.is_dir(), reason="bin directory not found") -def test_get_path(bin_path): +def test_get_path(): with add_sys_path(str(_bin_path)): ext, _ = get_suffixes(sys.platform) assert ( @@ -26,22 +21,22 @@ def test_get_path(bin_path): ) -def test_get_version(bin_path): - with add_sys_path(str(bin_path)): - ver_str = Executables.get_version("mf6", path=bin_path).partition(" ") +def test_get_version(): + with add_sys_path(str(_bin_path)): + ver_str = Executables.get_version("mf6", path=_bin_path).partition(" ") print(ver_str) version = int(ver_str[0].split(".")[0]) assert version >= 6 @pytest.fixture -def exes(bin_path): - return Executables(mf6=bin_path / f"mf6{_ext}") +def exes(): + return Executables(mf6=_bin_path / f"mf6{_ext}") -def test_executables_mapping(bin_path, exes): +def test_executables_mapping(exes): print(exes.mf6) - assert exes.mf6 == bin_path / f"mf6{_ext}" + assert exes.mf6 == _bin_path / f"mf6{_ext}" def test_executables_usage(exes): diff --git a/modflow_devtools/test/test_fixtures.py b/modflow_devtools/test/test_fixtures.py index b85da140..2a06e52c 100644 --- a/modflow_devtools/test/test_fixtures.py +++ b/modflow_devtools/test/test_fixtures.py @@ -254,9 +254,11 @@ def test_test_model_mf6(test_model_mf6): def test_test_model_mf5to6(test_model_mf5to6): assert isinstance(test_model_mf5to6, Path) - assert len(list(test_model_mf5to6.glob("*.nam"))) >= 1 + assert any(list(test_model_mf5to6.glob("*.nam"))) def test_large_test_model(large_test_model): assert isinstance(large_test_model, Path) - assert (large_test_model / "mfsim.nam").is_file() + assert (large_test_model / "mfsim.nam").is_file() or any( + list(large_test_model.glob("*.nam")) + ) diff --git a/modflow_devtools/test/test_misc.py b/modflow_devtools/test/test_misc.py index a89931e7..3f555270 100644 --- a/modflow_devtools/test/test_misc.py +++ b/modflow_devtools/test/test_misc.py @@ -1,2 +1,59 @@ -def test_set_dir(): - pass +import os +from os import environ +from pathlib import Path + +import pytest +from modflow_devtools.misc import get_model_paths, get_packages, set_dir + + +def test_set_dir(tmp_path): + assert Path(os.getcwd()) != tmp_path + with set_dir(tmp_path): + assert Path(os.getcwd()) == tmp_path + assert Path(os.getcwd()) != tmp_path + + +_repos_path = Path(environ.get("REPOS_PATH")).expanduser().absolute() +_examples_repo_path = _repos_path / "modflow6-examples" +_examples_path = _examples_repo_path / "examples" +_example_paths = ( + sorted(list(_examples_path.glob("ex-*"))) + if _examples_path.is_dir() + else [] +) + + +@pytest.mark.skipif(not any(_example_paths), reason="examples not found") +def test_has_packages(): + example_path = _example_paths[0] + packages = get_packages(example_path / "mfsim.nam") + assert set(packages) == {"tdis", "gwf", "ims"} + + +@pytest.mark.skipif(not any(_example_paths), reason="examples not found") +def test_get_model_paths(): + paths = get_model_paths(_examples_path) + assert len(paths) == 127 + + paths = get_model_paths(_examples_path, namefile="*.nam") + assert len(paths) == 339 + + +def test_get_model_paths_exclude_patterns(): + paths = get_model_paths(_examples_path, excluded=["gwt"]) + assert len(paths) == 63 + + +def test_get_model_paths_select_prefix(): + paths = get_model_paths(_examples_path, prefix="ex2") + assert not any(paths) + + +def test_get_model_paths_select_patterns(): + paths = get_model_paths(_examples_path, selected=["gwf"]) + assert len(paths) == 70 + + +def test_get_model_paths_select_packages(): + paths = get_model_paths(_examples_path, namefile="*.nam", packages=["wel"]) + assert len(paths) == 64