Gallerytest one fig per test

docs/gallery_tests/conftest.py

@@ -17,8 +17,7 @@
 GALLERY_DIR = CURRENT_DIR.parents[1] / "gallery_code"
 
 
-@pytest.fixture
-def image_setup_teardown():
+def _image_setup_teardown():
     """
     Setup and teardown fixture.
 
@@ -31,28 +30,7 @@ def image_setup_teardown():
     plt.close("all")
 
 
-@pytest.fixture
-def import_patches(monkeypatch):
-    """
-    Replace plt.show() with a function that does nothing, also add all the
-    gallery examples to sys.path.
-
-    """
-
-    def no_show():
-        pass
-
-    monkeypatch.setattr(plt, "show", no_show)
-
-    for example_dir in GALLERY_DIR.iterdir():
-        if example_dir.is_dir():
-            monkeypatch.syspath_prepend(example_dir)
-
-    yield
-
-
-@pytest.fixture
-def iris_future_defaults():
+def _iris_future_defaults():
     """
     Create a fixture which resets all the iris.FUTURE settings to the defaults,
     as otherwise changes made in one test can affect subsequent ones.
@@ -65,3 +43,35 @@ def iris_future_defaults():
         del default_future_kwargs[dead_option]
     with iris.FUTURE.context(**default_future_kwargs):
         yield
+
+
+# Make function and class scoped fixtures.
+image_setup_teardown = pytest.fixture(_image_setup_teardown)
+class_image_setup_teardown = pytest.fixture(
+    _image_setup_teardown, scope="class"
+)
+
+iris_future_defaults = pytest.fixture(_iris_future_defaults)
+class_iris_future_defaults = pytest.fixture(
+    _iris_future_defaults, scope="class"
+)
+
+
+@pytest.fixture(scope="module")
+def import_patches():
+    """
+    Replace plt.show() with a function that does nothing, also add all the
+    gallery examples to sys.path.  Done once for the whole test module.
+
+    """
+
+    def no_show():
+        pass
+
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setattr(plt, "show", no_show)
+        for example_dir in GALLERY_DIR.iterdir():
+            if example_dir.is_dir():
+                mp.syspath_prepend(example_dir)
+
+        yield

docs/gallery_tests/test_gallery_examples.py

@@ -14,6 +14,16 @@
 
 from .conftest import GALLERY_DIR
 
+TWO_FIG_EXAMPLES = [
+    "plot_atlantic_profiles",
+    "plot_cross_section",
+    "plot_lagged_ensemble",
+    "plot_wind_speed",
+    "plot_projections_and_annotations",
+]
+
+FOUR_FIG_EXAMPLES = ["plot_orca_projection", "plot_rotated_pole_mapping"]
+
 
 def gallery_examples():
     """Generator to yield all current gallery examples."""
@@ -22,8 +32,63 @@ def gallery_examples():
         yield example_file.stem
 
 
+def expected_figcounts(example_code):
+    """How many figures we think are in each example."""
+    if example_code in TWO_FIG_EXAMPLES:
+        return 2
+    elif example_code in FOUR_FIG_EXAMPLES:
+        return 4
+    else:
+        return 1
+
+
+def get_params():
+    """
+    Generator to yield sequence of (example, fig_number) pairs for gallery examples.
+    Every figure from the examples is represented by one pair (except for the lagged
+    ensemble example, which is handled separately).
+
+    """
+    for example in gallery_examples():
+        if example == "plot_lagged_ensemble":
+            continue
+        else:
+            for i in range(expected_figcounts(example)):
+                yield example, i
+
+
+# Make a class for the GloSea example as it's particularly slow running, so we
+# only want to run it once for the two tests.  Also define it before the other
+# tests so it is queued up first.
+@pytest.mark.xdist_group(name="group1")
+class TestLagged:
+    @pytest.fixture(scope="class")
+    def get_figures(
+        self,
+        class_image_setup_teardown,
+        class_iris_future_defaults,
+        import_patches,
+    ):
+
+        module = importlib.import_module("plot_lagged_ensemble")
+        module.main()
+
+        figs = [plt.figure(fig_num) for fig_num in plt.get_fignums()]
+        return figs
+
+    @pytest.mark.filterwarnings("error::iris.IrisDeprecation")
+    @pytest.mark.parametrize("fig_index", [0, 1], ids=lambda arg: f"fig{arg}")
+    def test_lagged_example(self, get_figures, fig_index):
+        assert len(get_figures) == 2
+        plt.figure(get_figures[fig_index])
+        image_id = f"gallery_tests.test_plot_lagged_ensemble.{fig_index}"
+        check_graphic(image_id)
+
+
 @pytest.mark.filterwarnings("error::iris.IrisDeprecation")
-@pytest.mark.parametrize("example", gallery_examples())
+@pytest.mark.parametrize(
+    "example", get_params(), ids=lambda arg: f"{arg[0]}-fig{arg[1]}"
+)
 def test_plot_example(
     example,
     image_setup_teardown,
@@ -32,13 +97,16 @@ def test_plot_example(
 ):
     """Test that all figures from example code match KGO."""
 
-    module = importlib.import_module(example)
+    example_code, fig_index = example
+    module = importlib.import_module(example_code)
 
     # Run example.
     module.main()
-    # Loop through open figures and set each to be the current figure so check_graphic
-    # will find it.
-    for fig_num in plt.get_fignums():
-        plt.figure(fig_num)
-        image_id = f"gallery_tests.test_{example}.{fig_num - 1}"
-        check_graphic(image_id, _RESULT_PATH)
+
+    # Sanity check we have the right number of figures.
+    assert len(plt.get_fignums()) == expected_figcounts(example_code)
+
+    # Compare chosen figure to KGO.
+    plt.figure(fig_index + 1)
+    image_id = f"gallery_tests.test_{example_code}.{fig_index}"
+    check_graphic(image_id)

lib/iris/tests/runner/_runner.py

@@ -131,6 +131,7 @@ def run(self):
         args = [
             None,
             f"-n={self.num_processors}",
+            "--dist=loadgroup",  # currently only gallery tests use grouping.
         ]
 
         if self.stop: