From c8786d23394154f1e39a5707c66dfc45bf31c754 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Fri, 11 Aug 2023 17:09:48 -0400 Subject: [PATCH 01/17] ci: smoke test with/out optional dependencies, mark more tests --- .github/workflows/commit.yml | 45 +++++++++++++++++++++++---------- autotest/test_mf6.py | 3 ++- autotest/test_model_splitter.py | 1 + 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index daf46852a5..bc2006f383 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -106,6 +106,8 @@ jobs: run: shell: bash timeout-minutes: 10 + env: + PYTHON_VERSION: 3.8 steps: - name: Checkout repo @@ -114,23 +116,40 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' cache-dependency-path: pyproject.toml - - name: Install Python dependencies + - name: Install core/test Python dependencies run: | pip install --upgrade pip pip install . - pip install ".[test, optional]" + pip install ".[test]" - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 + + - name: Smoke test (no opt deps) + working-directory: autotest + run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install optional dependencies + run: pip install ".[optional]" - - name: Run smoke tests - working-directory: ./autotest - run: | - pytest -v -n=auto --smoke --durations=0 --keep-failed=.failed + - name: Smoke test (all opt deps) + working-directory: autotest + run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Uninstall some opt deps + run: pip uninstall --yes pyproj shapely vtk + + - name: Smoke test (some opt deps) + working-directory: autotest + run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -138,9 +157,8 @@ jobs: uses: actions/upload-artifact@v3 if: failure() with: - name: failed-smoke-${{ matrix.os }}-${{ matrix.python-version }} - path: | - ./autotest/.failed/** + name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }} + path: ./autotest/.failed/** test: name: Test @@ -220,7 +238,7 @@ jobs: if: runner.os != 'Windows' working-directory: ./autotest run: | - pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile + pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -230,7 +248,7 @@ jobs: shell: bash -l {0} working-directory: ./autotest run: | - pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile + pytest -v -m="not example and not regression" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -244,8 +262,7 @@ jobs: ./autotest/.failed/** - name: Upload coverage - if: - github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') + if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') uses: codecov/codecov-action@v3 with: files: ./autotest/coverage.xml diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index 3a35108f2c..9774a775b5 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from modflow_devtools.markers import requires_exe +from modflow_devtools.markers import requires_exe, requires_pkg from modflow_devtools.misc import set_dir import flopy @@ -637,6 +637,7 @@ def test_binary_write(function_tmpdir, layered): @requires_exe("mf6") +@requires_pkg("shapely") @pytest.mark.parametrize("layered", [True, False]) def test_vor_binary_write(function_tmpdir, layered): # build voronoi grid diff --git a/autotest/test_model_splitter.py b/autotest/test_model_splitter.py index 08cbb42c5d..2644c3a6b7 100644 --- a/autotest/test_model_splitter.py +++ b/autotest/test_model_splitter.py @@ -200,6 +200,7 @@ def test_metis_splitting_with_lak_sfr(function_tmpdir): @requires_exe("mf6") +@requires_exe("pymetis") def test_save_load_node_mapping(function_tmpdir): sim_path = get_example_data_path() / "mf6-freyberg" new_sim_path = function_tmpdir / "mf6-freyberg/split_model" From 35fdd4d2e7184786ef9bd7b063c3ecd65c452bc7 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Fri, 11 Aug 2023 19:06:09 -0400 Subject: [PATCH 02/17] test(grid): check crs-related deprecation warnings only if pyproj avail --- autotest/test_grid.py | 20 ++++++++++++++------ autotest/test_gridgen.py | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/autotest/test_grid.py b/autotest/test_grid.py index c3dafe5557..3508dbca29 100644 --- a/autotest/test_grid.py +++ b/autotest/test_grid.py @@ -1,6 +1,7 @@ import os import re import warnings +from contextlib import nullcontext from warnings import warn import matplotlib @@ -594,9 +595,12 @@ def do_checks(g): do_checks(UnstructuredGrid(**d, crs=crs)) do_checks(VertexGrid(vertices=d["vertices"], crs=crs)) + # only check deprecations if pyproj is available + context = pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + # test deprecated 'epsg' parameter if isinstance(crs, int): - with pytest.deprecated_call(): + with context: do_checks(StructuredGrid(delr=delr, delc=delc, epsg=crs)) if HAS_PYPROJ and crs == 26916: @@ -612,14 +616,14 @@ def do_checks(g): do_checks(StructuredGrid(delr=delr, delc=delc, prjfile=prjfile)) # test deprecated 'prj' parameter - with pytest.deprecated_call(): + with context: do_checks(StructuredGrid(delr=delr, delc=delc, prj=prjfile)) # test deprecated 'proj4' parameter with warnings.catch_warnings(): warnings.simplefilter("ignore") # pyproj warning about conversion proj4 = crs_obj.to_proj4() - with pytest.deprecated_call(): + with context: do_checks(StructuredGrid(delr=delr, delc=delc, proj4=proj4)) @@ -675,9 +679,12 @@ def do_checks(g, *, exp_srs=expected_srs, exp_epsg=expected_epsg): sg.set_coord_info(crs=26915, merge_coord_info=False) do_checks(sg, exp_srs="EPSG:26915", exp_epsg=26915) + # only check deprecations if pyproj is available + context = pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + # test deprecated 'epsg' parameter if isinstance(crs, int): - with pytest.deprecated_call(): + with context: sg.set_coord_info(epsg=crs) do_checks(sg) @@ -827,8 +834,9 @@ def test_grid_crs_exceptions(): # test non-existing file not_a_file = "not-a-file" - with pytest.raises(FileNotFoundError): - StructuredGrid(delr=delr, delc=delc, prjfile=not_a_file) + if has_pkg("pyproj"): + with pytest.raises(FileNotFoundError): + StructuredGrid(delr=delr, delc=delc, prjfile=not_a_file) # note "sg.prjfile = not_a_file" intentionally does not raise anything # test unhandled keyword diff --git a/autotest/test_gridgen.py b/autotest/test_gridgen.py index 2453e473f1..0703a274eb 100644 --- a/autotest/test_gridgen.py +++ b/autotest/test_gridgen.py @@ -14,6 +14,7 @@ from flopy.utils.gridgen import Gridgen +@requires_exe("gridgen") def test_ctor_accepts_path_or_string(function_tmpdir): grid = GridCases().structured_small() From 732e8178ca8f477e8e557da8097909f54b0495ad Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Fri, 11 Aug 2023 19:22:19 -0400 Subject: [PATCH 03/17] more optional dependency testing fixes --- .github/workflows/commit.yml | 8 ++++---- autotest/test_export.py | 2 +- autotest/test_gridintersect.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index bc2006f383..d8ffd3bc59 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -129,7 +129,7 @@ jobs: - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 - - name: Smoke test (no opt deps) + - name: Smoke test (no optional dependencies) working-directory: autotest run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed env: @@ -138,16 +138,16 @@ jobs: - name: Install optional dependencies run: pip install ".[optional]" - - name: Smoke test (all opt deps) + - name: Smoke test (all optional dependencies) working-directory: autotest run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Uninstall some opt deps + - name: Uninstall some dependencies run: pip uninstall --yes pyproj shapely vtk - - name: Smoke test (some opt deps) + - name: Smoke test (some optional dependencies) working-directory: autotest run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed env: diff --git a/autotest/test_export.py b/autotest/test_export.py index 51e238f398..1495598106 100644 --- a/autotest/test_export.py +++ b/autotest/test_export.py @@ -1378,7 +1378,7 @@ def test_vtk_unstructured(function_tmpdir, example_data_path): assert np.allclose(np.ravel(top), top2), "Field data not properly written" -@requires_pkg("pyvista") +@requires_pkg("vtk", "pyvista") def test_vtk_to_pyvista(function_tmpdir, example_data_path): from autotest.test_mp7_cases import Mp7Cases diff --git a/autotest/test_gridintersect.py b/autotest/test_gridintersect.py index 82c1f93856..f4db5c11e7 100644 --- a/autotest/test_gridintersect.py +++ b/autotest/test_gridintersect.py @@ -1221,7 +1221,7 @@ def test_polygon_offset_rot_structured_grid_shapely(rtree): # %% test rasters -@requires_pkg("rasterstats", "scipy") +@requires_pkg("rasterstats", "scipy", "shapely") def test_rasters(example_data_path): ws = example_data_path / "options" raster_name = "dem.img" From 0a0f5cc17a5e79a81732fb34a377ef56b5f1ec1f Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Sat, 12 Aug 2023 13:41:35 -0400 Subject: [PATCH 04/17] use strict=True with has_pkg() --- autotest/test_grid.py | 2 +- autotest/test_gridintersect.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autotest/test_grid.py b/autotest/test_grid.py index 3508dbca29..ed1993b506 100644 --- a/autotest/test_grid.py +++ b/autotest/test_grid.py @@ -23,7 +23,7 @@ from flopy.utils.triangle import Triangle from flopy.utils.voronoi import VoronoiGrid -HAS_PYPROJ = has_pkg("pyproj") +HAS_PYPROJ = has_pkg("pyproj", strict=True) if HAS_PYPROJ: import pyproj diff --git a/autotest/test_gridintersect.py b/autotest/test_gridintersect.py index f4db5c11e7..498320e53b 100644 --- a/autotest/test_gridintersect.py +++ b/autotest/test_gridintersect.py @@ -14,7 +14,7 @@ from flopy.utils.gridintersect import GridIntersect from flopy.utils.triangle import Triangle -if has_pkg("shapely"): +if has_pkg("shapely", strict=True): from shapely.geometry import ( LineString, MultiLineString, From f7ac7a098f5ba3d7f763398b37c880ed34198bab Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Sat, 12 Aug 2023 14:10:34 -0400 Subject: [PATCH 05/17] fix test_grid.py --- autotest/test_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest/test_grid.py b/autotest/test_grid.py index ed1993b506..59a0c946ab 100644 --- a/autotest/test_grid.py +++ b/autotest/test_grid.py @@ -834,7 +834,7 @@ def test_grid_crs_exceptions(): # test non-existing file not_a_file = "not-a-file" - if has_pkg("pyproj"): + if HAS_PYPROJ: with pytest.raises(FileNotFoundError): StructuredGrid(delr=delr, delc=delc, prjfile=not_a_file) # note "sg.prjfile = not_a_file" intentionally does not raise anything From 675128a48486e0a957ecc2a07746bc26e3801084 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Sat, 12 Aug 2023 19:40:18 -0400 Subject: [PATCH 06/17] fix test_export.py --- autotest/test_export.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/autotest/test_export.py b/autotest/test_export.py index 1495598106..afe9bdf39a 100644 --- a/autotest/test_export.py +++ b/autotest/test_export.py @@ -52,6 +52,11 @@ from flopy.utils.geometry import Polygon +HAS_PYPROJ = has_pkg("pyproj", strict=True) +if HAS_PYPROJ: + import pyproj + + def namfiles() -> List[Path]: mf2005_path = get_example_data_path() / "mf2005_test" return list(mf2005_path.rglob("*.nam")) @@ -330,10 +335,7 @@ def test_write_gridlines_shapefile(function_tmpdir): for suffix in [".dbf", ".shp", ".shx"]: assert outshp.with_suffix(suffix).exists() - if has_pkg("pyproj"): - assert outshp.with_suffix(".prj").exists() - else: - assert not outshp.with_suffix(".prj").exists() + assert outshp.with_suffix(".prj").exists() == HAS_PYPROJ with shapefile.Reader(str(outshp)) as sf: assert sf.shapeType == shapefile.POLYLINE From 62d014a3a0fff729512b1a62518f8bbdbbd1224b Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 09:35:38 -0400 Subject: [PATCH 07/17] use matrix instead of sequential cmds (faster) --- .github/workflows/commit.yml | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index d8ffd3bc59..a4bfad1f82 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -108,6 +108,10 @@ jobs: timeout-minutes: 10 env: PYTHON_VERSION: 3.8 + strategy: + fail-fast: true + matrix: + optdeps: [ "all", "no", "some" ] steps: - name: Checkout repo @@ -120,36 +124,24 @@ jobs: cache: 'pip' cache-dependency-path: pyproject.toml - - name: Install core/test Python dependencies + - name: Install Python dependencies run: | pip install --upgrade pip pip install . pip install ".[test]" + if [[ "${{ matrix.optdeps }}" != "no" ]]; then + pip install ".[optional]" + fi + if [[ "${{ matrix.optdeps }}" == "some" ]]; then + pip uninstall --yes pyproj shapely vtk + fi - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 - - name: Smoke test (no optional dependencies) - working-directory: autotest - run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Install optional dependencies - run: pip install ".[optional]" - - - name: Smoke test (all optional dependencies) - working-directory: autotest - run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Uninstall some dependencies - run: pip uninstall --yes pyproj shapely vtk - - - name: Smoke test (some optional dependencies) + - name: Smoke test (${{ matrix.optdeps }} optional dependencies) working-directory: autotest - run: pytest -v -n=auto --smoke --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed + run: pytest -v -n=auto --smoke --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 958f5a680d58feac22d1d7d851caa72d15236b91 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 09:38:48 -0400 Subject: [PATCH 08/17] more descriptive job naming --- .github/workflows/commit.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index a4bfad1f82..ce249cf9b4 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -111,7 +111,10 @@ jobs: strategy: fail-fast: true matrix: - optdeps: [ "all", "no", "some" ] + optdeps: + - "all optional dependencies" + - "no optional dependencies" + - "some optional dependencies" steps: - name: Checkout repo @@ -129,17 +132,17 @@ jobs: pip install --upgrade pip pip install . pip install ".[test]" - if [[ "${{ matrix.optdeps }}" != "no" ]]; then + if [[ ! "${{ matrix.optdeps }}" == *"no"* ]]; then pip install ".[optional]" fi - if [[ "${{ matrix.optdeps }}" == "some" ]]; then + if [[ "${{ matrix.optdeps }}" == *"some"* ]]; then pip uninstall --yes pyproj shapely vtk fi - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 - - name: Smoke test (${{ matrix.optdeps }} optional dependencies) + - name: Smoke test (${{ matrix.optdeps }}) working-directory: autotest run: pytest -v -n=auto --smoke --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed env: From 5b3a335d52cb605d15827f1936906ccef7e8b00a Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 09:40:47 -0400 Subject: [PATCH 09/17] revisions --- autotest/test_grid.py | 16 ++++++++++------ autotest/test_model_splitter.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/autotest/test_grid.py b/autotest/test_grid.py index 59a0c946ab..214018777e 100644 --- a/autotest/test_grid.py +++ b/autotest/test_grid.py @@ -596,11 +596,13 @@ def do_checks(g): do_checks(VertexGrid(vertices=d["vertices"], crs=crs)) # only check deprecations if pyproj is available - context = pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + pyproj_avail_context = ( + pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + ) # test deprecated 'epsg' parameter if isinstance(crs, int): - with context: + with pyproj_avail_context: do_checks(StructuredGrid(delr=delr, delc=delc, epsg=crs)) if HAS_PYPROJ and crs == 26916: @@ -616,14 +618,14 @@ def do_checks(g): do_checks(StructuredGrid(delr=delr, delc=delc, prjfile=prjfile)) # test deprecated 'prj' parameter - with context: + with pyproj_avail_context: do_checks(StructuredGrid(delr=delr, delc=delc, prj=prjfile)) # test deprecated 'proj4' parameter with warnings.catch_warnings(): warnings.simplefilter("ignore") # pyproj warning about conversion proj4 = crs_obj.to_proj4() - with context: + with pyproj_avail_context: do_checks(StructuredGrid(delr=delr, delc=delc, proj4=proj4)) @@ -680,11 +682,13 @@ def do_checks(g, *, exp_srs=expected_srs, exp_epsg=expected_epsg): do_checks(sg, exp_srs="EPSG:26915", exp_epsg=26915) # only check deprecations if pyproj is available - context = pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + pyproj_avail_context = ( + pytest.deprecated_call() if HAS_PYPROJ else nullcontext() + ) # test deprecated 'epsg' parameter if isinstance(crs, int): - with context: + with pyproj_avail_context: sg.set_coord_info(epsg=crs) do_checks(sg) diff --git a/autotest/test_model_splitter.py b/autotest/test_model_splitter.py index 2644c3a6b7..8e8dfbe24b 100644 --- a/autotest/test_model_splitter.py +++ b/autotest/test_model_splitter.py @@ -200,7 +200,7 @@ def test_metis_splitting_with_lak_sfr(function_tmpdir): @requires_exe("mf6") -@requires_exe("pymetis") +@requires_pkg("pymetis") def test_save_load_node_mapping(function_tmpdir): sim_path = get_example_data_path() / "mf6-freyberg" new_sim_path = function_tmpdir / "mf6-freyberg/split_model" From 8b6c77d88d29824297aeaff2577993146cc54657 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 09:43:59 -0400 Subject: [PATCH 10/17] upload smoke test coverage --- .github/workflows/commit.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index ce249cf9b4..7e2e009684 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -154,6 +154,12 @@ jobs: with: name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }} path: ./autotest/.failed/** + + - name: Upload coverage + if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') + uses: codecov/codecov-action@v3 + with: + files: ./autotest/coverage.xml test: name: Test From e62973a1acfb9f4666073731db6958deb6e31b49 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 10:35:13 -0400 Subject: [PATCH 11/17] remove 3 random dependencies in partial fulfillment smoke test job * use current date as random seed * upload artifact reporting which deps were removed --- .github/workflows/commit.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 7e2e009684..c9600a7443 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -136,8 +136,18 @@ jobs: pip install ".[optional]" fi if [[ "${{ matrix.optdeps }}" == *"some"* ]]; then - pip uninstall --yes pyproj shapely vtk + deps=$(sed '/optional =/,/]/!d' pyproject.toml | sed -e '1d;$d' -e 's/\"//g' -e 's/,//g' | tr -d ' ' | cut -f 1 -d ';') + rmvd=$(echo $deps | tr ' ' '\n' | shuf --random-source <(yes date +%d.%m.%y) | head -n 3) + echo "Removing optional dependencies: $rmvd" >> removed_dependencies.txt + cat removed_dependencies.txt + pip uninstall --yes $rmvd fi + + - name: Upload removed dependencies log + uses: actions/upload-artifact@v3 + with: + name: smoke-test-removed-dependencies + path: ./removed_dependencies.txt - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 From 4dcf4a199fdb7d7dab925e4c4742680d64c104cd Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 10:47:08 -0400 Subject: [PATCH 12/17] mark test requiring scipy, don't fail fast --- .github/workflows/commit.yml | 6 +++++- autotest/test_mf6.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index c9600a7443..cdefd9cf05 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -109,7 +109,7 @@ jobs: env: PYTHON_VERSION: 3.8 strategy: - fail-fast: true + fail-fast: false matrix: optdeps: - "all optional dependencies" @@ -132,6 +132,10 @@ jobs: pip install --upgrade pip pip install . pip install ".[test]" + + # Install optional dependencies according to matrix.optdeps. + # If matrix.optdeps is "some" remove 3 optional dependencies + # selected randomly using the current date as the seed. if [[ ! "${{ matrix.optdeps }}" == *"no"* ]]; then pip install ".[optional]" fi diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index 9774a775b5..f9b475a4a7 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -637,7 +637,7 @@ def test_binary_write(function_tmpdir, layered): @requires_exe("mf6") -@requires_pkg("shapely") +@requires_pkg("shapely", "scipy") @pytest.mark.parametrize("layered", [True, False]) def test_vor_binary_write(function_tmpdir, layered): # build voronoi grid From 088047be1a221ab1071e799bf2723a5fe1dae7e6 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 11:12:57 -0400 Subject: [PATCH 13/17] don't upload coverage from smoke tests (for now?) --- .github/workflows/commit.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index cdefd9cf05..c3d498c352 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -169,11 +169,11 @@ jobs: name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }} path: ./autotest/.failed/** - - name: Upload coverage - if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') - uses: codecov/codecov-action@v3 - with: - files: ./autotest/coverage.xml + # - name: Upload coverage + # if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') + # uses: codecov/codecov-action@v3 + # with: + # files: ./autotest/coverage.xml test: name: Test From 719af7f1a5752bd8a7288cbf2b1b497b3b2493f0 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 14 Aug 2023 22:31:39 -0400 Subject: [PATCH 14/17] move optional dependency tests to nightly schedule --- .github/workflows/commit.yml | 35 +++------------ .github/workflows/optional.yml | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/optional.yml diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index c3d498c352..18075f5577 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -108,13 +108,6 @@ jobs: timeout-minutes: 10 env: PYTHON_VERSION: 3.8 - strategy: - fail-fast: false - matrix: - optdeps: - - "all optional dependencies" - - "no optional dependencies" - - "some optional dependencies" steps: - name: Checkout repo @@ -131,21 +124,7 @@ jobs: run: | pip install --upgrade pip pip install . - pip install ".[test]" - - # Install optional dependencies according to matrix.optdeps. - # If matrix.optdeps is "some" remove 3 optional dependencies - # selected randomly using the current date as the seed. - if [[ ! "${{ matrix.optdeps }}" == *"no"* ]]; then - pip install ".[optional]" - fi - if [[ "${{ matrix.optdeps }}" == *"some"* ]]; then - deps=$(sed '/optional =/,/]/!d' pyproject.toml | sed -e '1d;$d' -e 's/\"//g' -e 's/,//g' | tr -d ' ' | cut -f 1 -d ';') - rmvd=$(echo $deps | tr ' ' '\n' | shuf --random-source <(yes date +%d.%m.%y) | head -n 3) - echo "Removing optional dependencies: $rmvd" >> removed_dependencies.txt - cat removed_dependencies.txt - pip uninstall --yes $rmvd - fi + pip install ".[test,optional]" - name: Upload removed dependencies log uses: actions/upload-artifact@v3 @@ -156,7 +135,7 @@ jobs: - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 - - name: Smoke test (${{ matrix.optdeps }}) + - name: Smoke test working-directory: autotest run: pytest -v -n=auto --smoke --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed env: @@ -169,11 +148,11 @@ jobs: name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }} path: ./autotest/.failed/** - # - name: Upload coverage - # if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') - # uses: codecov/codecov-action@v3 - # with: - # files: ./autotest/coverage.xml + - name: Upload coverage + if: github.repository_owner == 'modflowpy' && (github.event_name == 'push' || github.event_name == 'pull_request') + uses: codecov/codecov-action@v3 + with: + files: ./autotest/coverage.xml test: name: Test diff --git a/.github/workflows/optional.yml b/.github/workflows/optional.yml new file mode 100644 index 0000000000..96d37497a1 --- /dev/null +++ b/.github/workflows/optional.yml @@ -0,0 +1,79 @@ +name: FloPy optional dependency testing +on: + pull_request: + branches: + - master + - develop + schedule: + - cron: '0 8 * * *' # run at 8 AM UTC (12 am PST) +jobs: + test: + name: Test + runs-on: ubuntu-latest + defaults: + run: + shell: bash + timeout-minutes: 10 + env: + PYTHON_VERSION: 3.8 + strategy: + fail-fast: false + matrix: + optdeps: + - "all optional dependencies" + - "no optional dependencies" + - "some optional dependencies" + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: pyproject.toml + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install . + pip install ".[test]" + + # Install optional dependencies according to matrix.optdeps. + # If matrix.optdeps is "some" remove 3 optional dependencies + # selected randomly using the current date as the seed. + if [[ ! "${{ matrix.optdeps }}" == *"no"* ]]; then + pip install ".[optional]" + fi + if [[ "${{ matrix.optdeps }}" == *"some"* ]]; then + deps=$(sed '/optional =/,/]/!d' pyproject.toml | sed -e '1d;$d' -e 's/\"//g' -e 's/,//g' | tr -d ' ' | cut -f 1 -d ';') + rmvd=$(echo $deps | tr ' ' '\n' | shuf --random-source <(yes date +%d.%m.%y) | head -n 3) + echo "Removing optional dependencies: $rmvd" >> removed_dependencies.txt + cat removed_dependencies.txt + pip uninstall --yes $rmvd + fi + + - name: Upload removed dependencies log + uses: actions/upload-artifact@v3 + with: + name: smoke-test-removed-dependencies + path: ./removed_dependencies.txt + + - name: Install Modflow executables + uses: modflowpy/install-modflow-action@v1 + + - name: Smoke test (${{ matrix.optdeps }}) + working-directory: autotest + run: pytest -v -n=auto --smoke --cov=flopy --cov-report=xml --durations=0 --keep-failed=.failed + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload failed test outputs + uses: actions/upload-artifact@v3 + if: failure() + with: + name: failed-smoke-${{ runner.os }}-${{ env.PYTHON_VERSION }} + path: ./autotest/.failed/** + \ No newline at end of file From 9e49798e46a7e156b1371385f053d8a4afc1cb06 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Tue, 15 Aug 2023 00:12:24 -0400 Subject: [PATCH 15/17] deduplicate --- .github/workflows/optional.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/optional.yml b/.github/workflows/optional.yml index 96d37497a1..f8c1618d67 100644 --- a/.github/workflows/optional.yml +++ b/.github/workflows/optional.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: optdeps: - - "all optional dependencies" + # - "all optional dependencies" - "no optional dependencies" - "some optional dependencies" From 1c0b89e7daa71ada3e055d1f7519cf3d97d8b234 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Tue, 15 Aug 2023 00:58:40 -0400 Subject: [PATCH 16/17] fix optional.yml triggers --- .github/workflows/optional.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/optional.yml b/.github/workflows/optional.yml index f8c1618d67..094c92e6c4 100644 --- a/.github/workflows/optional.yml +++ b/.github/workflows/optional.yml @@ -1,9 +1,5 @@ name: FloPy optional dependency testing on: - pull_request: - branches: - - master - - develop schedule: - cron: '0 8 * * *' # run at 8 AM UTC (12 am PST) jobs: From 424d8141bde9e442b5c9a09ceeb578523e6ae525 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Tue, 15 Aug 2023 22:04:13 -0400 Subject: [PATCH 17/17] remove unneeded step --- .github/workflows/commit.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 18075f5577..770eb2460b 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -126,12 +126,6 @@ jobs: pip install . pip install ".[test,optional]" - - name: Upload removed dependencies log - uses: actions/upload-artifact@v3 - with: - name: smoke-test-removed-dependencies - path: ./removed_dependencies.txt - - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1