From dbda763cd42e0e5ba183fa78312e9f4afd8e437e Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 11:41:07 +0100 Subject: [PATCH 1/7] first pass class for GloSea example to reduce runtime this class addition brings total run time of gallery tests from ~3:40 to ~2:35 (total time before this branch ~2:20). Timings on ancient laptop. tidyings --- docs/gallery_tests/conftest.py | 58 +++++++++------- docs/gallery_tests/test_gallery_examples.py | 74 ++++++++++++++++++--- 2 files changed, 99 insertions(+), 33 deletions(-) diff --git a/docs/gallery_tests/conftest.py b/docs/gallery_tests/conftest.py index a218b305a2..583ac3db27 100644 --- a/docs/gallery_tests/conftest.py +++ b/docs/gallery_tests/conftest.py @@ -17,8 +17,7 @@ GALLERY_DIR = CURRENT_DIR.parents[1] / "gallery_code" -@pytest.fixture -def image_setup_teardown(): +def _image_setup_teardown(): """ Setup and teardown fixture. @@ -31,28 +30,7 @@ def image_setup_teardown(): plt.close("all") -@pytest.fixture -def import_patches(monkeypatch): - """ - Replace plt.show() with a function that does nothing, also add all the - gallery examples to sys.path. - - """ - - def no_show(): - pass - - monkeypatch.setattr(plt, "show", no_show) - - for example_dir in GALLERY_DIR.iterdir(): - if example_dir.is_dir(): - monkeypatch.syspath_prepend(example_dir) - - yield - - -@pytest.fixture -def iris_future_defaults(): +def _iris_future_defaults(): """ Create a fixture which resets all the iris.FUTURE settings to the defaults, as otherwise changes made in one test can affect subsequent ones. @@ -65,3 +43,35 @@ def iris_future_defaults(): del default_future_kwargs[dead_option] with iris.FUTURE.context(**default_future_kwargs): yield + + +# Make function and class scoped fixtures. +image_setup_teardown = pytest.fixture(_image_setup_teardown) +class_image_setup_teardown = pytest.fixture( + _image_setup_teardown, scope="class" +) + +iris_future_defaults = pytest.fixture(_iris_future_defaults) +class_iris_future_defaults = pytest.fixture( + _iris_future_defaults, scope="class" +) + + +@pytest.fixture(scope="module") +def monkeypatching(): + """ + Replace plt.show() with a function that does nothing, also add all the + gallery examples to sys.path. Done once for the whole test module. + + """ + + def no_show(): + pass + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(plt, "show", no_show) + for example_dir in GALLERY_DIR.iterdir(): + if example_dir.is_dir(): + mp.syspath_prepend(example_dir) + + yield diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index 0d0793a7da..3abbf70249 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -14,31 +14,87 @@ from .conftest import GALLERY_DIR +TWO_FIG_EXAMPLES = [ + "plot_atlantic_profiles", + "plot_cross_section", + "plot_lagged_ensemble", + "plot_wind_speed", + "plot_projections_and_annotations", +] + +FOUR_FIG_EXAMPLES = ["plot_orca_projection", "rotated_pole_mapping"] + def gallery_examples(): """Generator to yield all current gallery examples.""" - + for example_file in GALLERY_DIR.glob("*/plot*.py"): yield example_file.stem +def get_params(): + """ + Generator to yield sequence of (example, fig_number) pairs for gallery examples. + Every figure from the examples is represented by one pair (except for the lagged + ensemble example, which is handled separately). + + """ + for example in gallery_examples(): + if example == "plot_lagged_ensemble": + continue + elif example in TWO_FIG_EXAMPLES: + for i in range(2): + yield example, i + elif example in FOUR_FIG_EXAMPLES: + for i in range(4): + yield example, i + else: + yield example, 0 + + @pytest.mark.filterwarnings("error::iris.IrisDeprecation") -@pytest.mark.parametrize("example", gallery_examples()) +@pytest.mark.parametrize( + "example", get_params(), ids=lambda arg: f"{arg[0]}-fig{arg[1]}" +) def test_plot_example( example, image_setup_teardown, import_patches, iris_future_defaults, + monkeypatching, ): """Test that all figures from example code match KGO.""" - module = importlib.import_module(example) + example_code, fig_index = example + module = importlib.import_module(example_code) # Run example. module.main() - # Loop through open figures and set each to be the current figure so check_graphic - # will find it. - for fig_num in plt.get_fignums(): - plt.figure(fig_num) - image_id = f"gallery_tests.test_{example}.{fig_num - 1}" - check_graphic(image_id, _RESULT_PATH) + plt.figure(fig_index + 1) + image_id = f"gallery_tests.test_{example_code}.{fig_index}" + check_graphic(image_id) + + +# Make a class for the GloSea example as it's particularly slow running, so we +# only want to run it once for the two tests. +class TestLagged: + @pytest.fixture(scope="class") + def get_figures( + self, + class_image_setup_teardown, + class_iris_future_defaults, + monkeypatching, + ): + + module = importlib.import_module("plot_lagged_ensemble") + module.main() + + figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()] + return figs + + @pytest.mark.filterwarnings("error::iris.IrisDeprecation") + @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") + def test_lagged_example(self, get_figures, fig_index): + plt.figure(get_figures[fig_index]) + image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" + check_graphic(image_id) From cdbbc710e83e38d5996eff3d3055b8c191718bba Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 14:29:43 +0100 Subject: [PATCH 2/7] check we have correct number of figs --- docs/gallery_tests/test_gallery_examples.py | 27 +++++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index 3abbf70249..d87bcbb191 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -22,7 +22,7 @@ "plot_projections_and_annotations", ] -FOUR_FIG_EXAMPLES = ["plot_orca_projection", "rotated_pole_mapping"] +FOUR_FIG_EXAMPLES = ["plot_orca_projection", "plot_rotated_pole_mapping"] def gallery_examples(): @@ -32,6 +32,16 @@ def gallery_examples(): yield example_file.stem +def expected_fignums(example_code): + """How many figures we think are in each example.""" + if example_code in TWO_FIG_EXAMPLES: + return 2 + elif example_code in FOUR_FIG_EXAMPLES: + return 4 + else: + return 1 + + def get_params(): """ Generator to yield sequence of (example, fig_number) pairs for gallery examples. @@ -42,14 +52,9 @@ def get_params(): for example in gallery_examples(): if example == "plot_lagged_ensemble": continue - elif example in TWO_FIG_EXAMPLES: - for i in range(2): - yield example, i - elif example in FOUR_FIG_EXAMPLES: - for i in range(4): - yield example, i else: - yield example, 0 + for i in range(expected_fignums(example)): + yield example, i @pytest.mark.filterwarnings("error::iris.IrisDeprecation") @@ -70,6 +75,11 @@ def test_plot_example( # Run example. module.main() + + # Sanity check we have the right number of figures. + assert len(plt.get_fignums()) == expected_fignums(example_code) + + # Compare chosen figure to KGO. plt.figure(fig_index + 1) image_id = f"gallery_tests.test_{example_code}.{fig_index}" check_graphic(image_id) @@ -95,6 +105,7 @@ def get_figures( @pytest.mark.filterwarnings("error::iris.IrisDeprecation") @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") def test_lagged_example(self, get_figures, fig_index): + assert len(get_figures) == 2 plt.figure(get_figures[fig_index]) image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" check_graphic(image_id) From 39ec96dfad6c418823a2ec6a240c9dae432a1d4d Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 14:53:16 +0100 Subject: [PATCH 3/7] get lagged ensemble example onto single processor and running first --- docs/gallery_tests/test_gallery_examples.py | 54 +++++++++++---------- lib/iris/tests/runner/_runner.py | 1 + 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index d87bcbb191..5c6afdf099 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -57,6 +57,34 @@ def get_params(): yield example, i +# Make a class for the GloSea example as it's particularly slow running, so we +# only want to run it once for the two tests. Also define it before the other +# tests so it is queued up first. +@pytest.mark.xdist_group(name="group1") +class TestLagged: + @pytest.fixture(scope="class") + def get_figures( + self, + class_image_setup_teardown, + class_iris_future_defaults, + monkeypatching, + ): + + module = importlib.import_module("plot_lagged_ensemble") + module.main() + + figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()] + return figs + + @pytest.mark.filterwarnings("error::iris.IrisDeprecation") + @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") + def test_lagged_example(self, get_figures, fig_index): + assert len(get_figures) == 2 + plt.figure(get_figures[fig_index]) + image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" + check_graphic(image_id) + + @pytest.mark.filterwarnings("error::iris.IrisDeprecation") @pytest.mark.parametrize( "example", get_params(), ids=lambda arg: f"{arg[0]}-fig{arg[1]}" @@ -83,29 +111,3 @@ def test_plot_example( plt.figure(fig_index + 1) image_id = f"gallery_tests.test_{example_code}.{fig_index}" check_graphic(image_id) - - -# Make a class for the GloSea example as it's particularly slow running, so we -# only want to run it once for the two tests. -class TestLagged: - @pytest.fixture(scope="class") - def get_figures( - self, - class_image_setup_teardown, - class_iris_future_defaults, - monkeypatching, - ): - - module = importlib.import_module("plot_lagged_ensemble") - module.main() - - figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()] - return figs - - @pytest.mark.filterwarnings("error::iris.IrisDeprecation") - @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") - def test_lagged_example(self, get_figures, fig_index): - assert len(get_figures) == 2 - plt.figure(get_figures[fig_index]) - image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" - check_graphic(image_id) diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py index bfb2cc2402..72442dd6d5 100644 --- a/lib/iris/tests/runner/_runner.py +++ b/lib/iris/tests/runner/_runner.py @@ -131,6 +131,7 @@ def run(self): args = [ None, f"-n={self.num_processors}", + "--dist=loadgroup", # currently only gallery tests use grouping. ] if self.stop: From 7d0115554f0a04dcff61f1312413efefe32d75de Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 14:59:25 +0100 Subject: [PATCH 4/7] better function names --- docs/gallery_tests/conftest.py | 2 +- docs/gallery_tests/test_gallery_examples.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/gallery_tests/conftest.py b/docs/gallery_tests/conftest.py index 583ac3db27..1fe12aa3dd 100644 --- a/docs/gallery_tests/conftest.py +++ b/docs/gallery_tests/conftest.py @@ -58,7 +58,7 @@ def _iris_future_defaults(): @pytest.fixture(scope="module") -def monkeypatching(): +def import_patching(): """ Replace plt.show() with a function that does nothing, also add all the gallery examples to sys.path. Done once for the whole test module. diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index 5c6afdf099..0b68d8e15d 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -32,7 +32,7 @@ def gallery_examples(): yield example_file.stem -def expected_fignums(example_code): +def expected_figcounts(example_code): """How many figures we think are in each example.""" if example_code in TWO_FIG_EXAMPLES: return 2 @@ -53,7 +53,7 @@ def get_params(): if example == "plot_lagged_ensemble": continue else: - for i in range(expected_fignums(example)): + for i in range(expected_figcounts(example)): yield example, i @@ -67,7 +67,7 @@ def get_figures( self, class_image_setup_teardown, class_iris_future_defaults, - monkeypatching, + import_patching, ): module = importlib.import_module("plot_lagged_ensemble") @@ -94,7 +94,7 @@ def test_plot_example( image_setup_teardown, import_patches, iris_future_defaults, - monkeypatching, + import_patching, ): """Test that all figures from example code match KGO.""" @@ -105,7 +105,7 @@ def test_plot_example( module.main() # Sanity check we have the right number of figures. - assert len(plt.get_fignums()) == expected_fignums(example_code) + assert len(plt.get_fignums()) == expected_figcounts(example_code) # Compare chosen figure to KGO. plt.figure(fig_index + 1) From f2798bc51167cd5d6299defd5ccc679d9d6d1eaa Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 15:19:13 +0100 Subject: [PATCH 5/7] remove GloSea class --- docs/gallery_tests/test_gallery_examples.py | 38 ++------------------- lib/iris/tests/runner/_runner.py | 1 - 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index 0b68d8e15d..bb4ccd0431 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -45,44 +45,12 @@ def expected_figcounts(example_code): def get_params(): """ Generator to yield sequence of (example, fig_number) pairs for gallery examples. - Every figure from the examples is represented by one pair (except for the lagged - ensemble example, which is handled separately). + Every figure from the examples is represented by one pair. """ for example in gallery_examples(): - if example == "plot_lagged_ensemble": - continue - else: - for i in range(expected_figcounts(example)): - yield example, i - - -# Make a class for the GloSea example as it's particularly slow running, so we -# only want to run it once for the two tests. Also define it before the other -# tests so it is queued up first. -@pytest.mark.xdist_group(name="group1") -class TestLagged: - @pytest.fixture(scope="class") - def get_figures( - self, - class_image_setup_teardown, - class_iris_future_defaults, - import_patching, - ): - - module = importlib.import_module("plot_lagged_ensemble") - module.main() - - figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()] - return figs - - @pytest.mark.filterwarnings("error::iris.IrisDeprecation") - @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") - def test_lagged_example(self, get_figures, fig_index): - assert len(get_figures) == 2 - plt.figure(get_figures[fig_index]) - image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" - check_graphic(image_id) + for i in range(expected_figcounts(example)): + yield example, i @pytest.mark.filterwarnings("error::iris.IrisDeprecation") diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py index 72442dd6d5..bfb2cc2402 100644 --- a/lib/iris/tests/runner/_runner.py +++ b/lib/iris/tests/runner/_runner.py @@ -131,7 +131,6 @@ def run(self): args = [ None, f"-n={self.num_processors}", - "--dist=loadgroup", # currently only gallery tests use grouping. ] if self.stop: From 11c6338de0328e12747a9196d5df1e407d5c621c Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Sun, 19 Jun 2022 15:36:23 +0100 Subject: [PATCH 6/7] reinstate GloSea class This reverts commit 814251d239f504fdbb37bc58770ce743160577c0. --- docs/gallery_tests/test_gallery_examples.py | 38 +++++++++++++++++++-- lib/iris/tests/runner/_runner.py | 1 + 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index bb4ccd0431..0b68d8e15d 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -45,12 +45,44 @@ def expected_figcounts(example_code): def get_params(): """ Generator to yield sequence of (example, fig_number) pairs for gallery examples. - Every figure from the examples is represented by one pair. + Every figure from the examples is represented by one pair (except for the lagged + ensemble example, which is handled separately). """ for example in gallery_examples(): - for i in range(expected_figcounts(example)): - yield example, i + if example == "plot_lagged_ensemble": + continue + else: + for i in range(expected_figcounts(example)): + yield example, i + + +# Make a class for the GloSea example as it's particularly slow running, so we +# only want to run it once for the two tests. Also define it before the other +# tests so it is queued up first. +@pytest.mark.xdist_group(name="group1") +class TestLagged: + @pytest.fixture(scope="class") + def get_figures( + self, + class_image_setup_teardown, + class_iris_future_defaults, + import_patching, + ): + + module = importlib.import_module("plot_lagged_ensemble") + module.main() + + figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()] + return figs + + @pytest.mark.filterwarnings("error::iris.IrisDeprecation") + @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}") + def test_lagged_example(self, get_figures, fig_index): + assert len(get_figures) == 2 + plt.figure(get_figures[fig_index]) + image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}" + check_graphic(image_id) @pytest.mark.filterwarnings("error::iris.IrisDeprecation") diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py index bfb2cc2402..72442dd6d5 100644 --- a/lib/iris/tests/runner/_runner.py +++ b/lib/iris/tests/runner/_runner.py @@ -131,6 +131,7 @@ def run(self): args = [ None, f"-n={self.num_processors}", + "--dist=loadgroup", # currently only gallery tests use grouping. ] if self.stop: From c7b3ab0d683101585beebc721a36fa84268febd7 Mon Sep 17 00:00:00 2001 From: Ruth Comer Date: Mon, 20 Jun 2022 18:03:15 +0100 Subject: [PATCH 7/7] post rebase tidy-up --- docs/gallery_tests/conftest.py | 2 +- docs/gallery_tests/test_gallery_examples.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/gallery_tests/conftest.py b/docs/gallery_tests/conftest.py index 1fe12aa3dd..808233a5ed 100644 --- a/docs/gallery_tests/conftest.py +++ b/docs/gallery_tests/conftest.py @@ -58,7 +58,7 @@ def _iris_future_defaults(): @pytest.fixture(scope="module") -def import_patching(): +def import_patches(): """ Replace plt.show() with a function that does nothing, also add all the gallery examples to sys.path. Done once for the whole test module. diff --git a/docs/gallery_tests/test_gallery_examples.py b/docs/gallery_tests/test_gallery_examples.py index 0b68d8e15d..116a0d9087 100644 --- a/docs/gallery_tests/test_gallery_examples.py +++ b/docs/gallery_tests/test_gallery_examples.py @@ -27,7 +27,7 @@ def gallery_examples(): """Generator to yield all current gallery examples.""" - + for example_file in GALLERY_DIR.glob("*/plot*.py"): yield example_file.stem @@ -67,7 +67,7 @@ def get_figures( self, class_image_setup_teardown, class_iris_future_defaults, - import_patching, + import_patches, ): module = importlib.import_module("plot_lagged_ensemble") @@ -94,7 +94,6 @@ def test_plot_example( image_setup_teardown, import_patches, iris_future_defaults, - import_patching, ): """Test that all figures from example code match KGO."""