ENH: Parallel example execution in doc build

.circleci/config.yml

@@ -27,8 +27,8 @@ jobs:
       scheduled:
         type: string
         default: "false"
-    docker:
-      - image: cimg/base:current-22.04
+    machine:
+      image: ubuntu-2404:current
     # large 4 vCPUs 15GB mem
     # https://discuss.circleci.com/t/changes-to-remote-docker-reporting-pricing/47759
     resource_class: large
@@ -111,7 +111,7 @@ jobs:
       # Load pip cache
       - restore_cache:
           keys:
-            - pip-cache
+            - pip-cache-0
       - restore_cache:
           keys:
             - user-install-bin-cache-310
@@ -123,7 +123,7 @@ jobs:
             ./tools/circleci_dependencies.sh
 
       - save_cache:
-          key: pip-cache
+          key: pip-cache-0
           paths:
             - ~/.cache/pip
       - save_cache:
@@ -422,7 +422,7 @@ jobs:
           command: ./tools/circleci_bash_env.sh
       - restore_cache:
           keys:
-            - pip-cache
+            - pip-cache-0
       - run:
           name: Get Python running
           command: |

doc/conf.py

@@ -15,7 +15,6 @@
 from pathlib import Path
 
 import matplotlib
-import pyvista
 import sphinx
 from intersphinx_registry import get_intersphinx_mapping
 from numpydoc import docscrape
@@ -443,10 +442,6 @@
 examples_dirs = ["../tutorials", "../examples"]
 gallery_dirs = ["auto_tutorials", "auto_examples"]
 os.environ["_MNE_BUILDING_DOC"] = "true"
-scrapers = ("matplotlib",)
-mne.viz.set_3d_backend("pyvistaqt")
-pyvista.OFF_SCREEN = False
-pyvista.BUILDING_GALLERY = True
 
 scrapers = (
     "matplotlib",
@@ -466,6 +461,7 @@
     except Exception:
         compress_images = ()
 
+sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1"))
 sphinx_gallery_conf = {
     "doc_module": ("mne",),
     "reference_url": dict(mne=None),
@@ -517,7 +513,7 @@
     ),  # called w/each script
     "reset_modules_order": "both",
     "image_scrapers": scrapers,
-    "show_memory": not sys.platform.startswith(("win", "darwin")),
+    "show_memory": sys.platform == "linux" and sphinx_gallery_parallel == 1,
     "line_numbers": False,  # messes with style
     "within_subsection_order": "FileNameSortKey",
     "capture_repr": ("_repr_html_",),
@@ -565,6 +561,7 @@
         ".*.remove.*|.*.write.*)"
     ),
     "copyfile_regex": r".*index\.rst",  # allow custom index.rst files
+    "parallel": sphinx_gallery_parallel,
 }
 assert is_serializable(sphinx_gallery_conf)
 # Files were renamed from plot_* with:
@@ -1650,8 +1647,8 @@ def make_version(app, exception):
 def setup(app):
     """Set up the Sphinx app."""
     app.connect("autodoc-process-docstring", append_attr_meth_examples)
-    report_scraper.app = app
-    app.connect("builder-inited", report_scraper.copyfiles)
+    # High prio, will happen before SG
+    app.connect("builder-inited", report_scraper.set_dirs, priority=20)
     app.connect("build-finished", make_gallery_redirects)
     app.connect("build-finished", make_api_redirects)
     app.connect("build-finished", make_custom_redirects)

doc/sphinxext/mne_doc_utils.py

@@ -6,6 +6,7 @@
 import warnings
 
 import numpy as np
+import pyvista
 
 import mne
 from mne.utils import (
@@ -86,6 +87,8 @@ def reset_warnings(gallery_conf, fname):
         r"open_text is deprecated\. Use files.*",
         # python-quantities, via neo
         r"numpy\.core is deprecated and has been renamed to numpy\._core",
+        # matplotlib
+        "__array_wrap__ must accept context and return_scalar.*",
     ):
         warnings.filterwarnings(  # deal with other modules having bad imports
             "ignore", message=f".*{key}.*", category=DeprecationWarning
@@ -146,6 +149,12 @@ def reset_warnings(gallery_conf, fname):
         "ignore",
         r"mne\.io\.pick.channel_indices_by_type is deprecated.*",
     )
+    # parallel building
+    warnings.filterwarnings(
+        "ignore",
+        "A worker stopped while some jobs were given to the executor.*",
+        category=UserWarning,
+    )
 
     # In case we use np.set_printoptions in any tutorials, we only
     # want it to affect those:
@@ -159,10 +168,12 @@ def reset_modules(gallery_conf, fname, when):
     """Do the reset."""
     import matplotlib.pyplot as plt
 
-    try:
-        from pyvista import Plotter  # noqa
-    except ImportError:
-        Plotter = None  # noqa
+    mne.viz.set_3d_backend("pyvistaqt")
+    pyvista.OFF_SCREEN = False
+    pyvista.BUILDING_GALLERY = True
+
+    from pyvista import Plotter  # noqa
+
     try:
         from pyvistaqt import BackgroundPlotter  # noqa
     except ImportError:

mne/report/report.py

@@ -4303,10 +4303,6 @@ class _ReportScraper:
     is written to the same directory as the example script.
     """
 
-    def __init__(self):
-        self.app = None
-        self.files = dict()
-
     def __repr__(self):
         return "<ReportScraper>"
 
@@ -4325,25 +4321,25 @@ def __call__(self, block, block_vars, gallery_conf):
                 with open(img_fname, "w") as fid:
                     fid.write(_FA_FILE_CODE)
                 # copy HTML file
-                html_fname = op.basename(report.fname)
-                out_dir = op.join(
-                    self.app.builder.outdir,
-                    op.relpath(
-                        op.dirname(block_vars["target_file"]), self.app.builder.srcdir
-                    ),
+                html_fname = Path(report.fname).name
+                srcdir = Path(gallery_conf["src_dir"])
+                outdir = Path(gallery_conf["out_dir"])
+                out_dir = outdir / Path(block_vars["target_file"]).parent.relative_to(
+                    srcdir
                 )
                 os.makedirs(out_dir, exist_ok=True)
-                out_fname = op.join(out_dir, html_fname)
+                out_fname = out_dir / html_fname
+                copyfile(report.fname, out_fname)
                 assert op.isfile(report.fname)
-                self.files[report.fname] = out_fname
                 # embed links/iframe
                 data = _SCRAPER_TEXT.format(html_fname)
                 return data
         return ""
 
-    def copyfiles(self, *args, **kwargs):
-        for key, value in self.files.items():
-            copyfile(key, value)
+    def set_dirs(self, app):
+        # Inject something into sphinx_gallery_conf as this gets pickled properly
+        # during parallel example generation
+        app.config.sphinx_gallery_conf["out_dir"] = app.builder.outdir
 
 
 def _df_bootstrap_table(*, df, data_id):

mne/report/tests/test_report.py

@@ -792,14 +792,18 @@ def test_scraper(tmp_path):
     r.add_figure(fig=fig1, title="a")
     r.add_figure(fig=fig2, title="b")
     # Mock a Sphinx + sphinx_gallery config
-    app = Bunch(builder=Bunch(srcdir=tmp_path, outdir=tmp_path / "_build" / "html"))
+    srcdir = tmp_path
+    outdir = tmp_path / "_build" / "html"
     scraper = _ReportScraper()
-    scraper.app = app
-    gallery_conf = dict(src_dir=app.builder.srcdir, builder_name="html")
-    img_fname = app.builder.srcdir / "auto_examples" / "images" / "sg_img.png"
-    target_file = app.builder.srcdir / "auto_examples" / "sg.py"
+    gallery_conf = dict(builder_name="html", src_dir=srcdir)
+    app = Bunch(
+        builder=Bunch(outdir=outdir),
+        config=Bunch(sphinx_gallery_conf=gallery_conf),
+    )
+    scraper.set_dirs(app)
+    img_fname = srcdir / "auto_examples" / "images" / "sg_img.png"
+    target_file = srcdir / "auto_examples" / "sg.py"
     os.makedirs(img_fname.parent)
-    os.makedirs(app.builder.outdir)
     block_vars = dict(
         image_path_iterator=(img for img in [str(img_fname)]),
         example_globals=dict(a=1),
@@ -814,12 +818,11 @@ def test_scraper(tmp_path):
     rst = scraper(block, block_vars, gallery_conf)
     # Once it's saved, add it
     assert rst == ""
-    fname = tmp_path / "my_html.html"
+    fname = srcdir / "my_html.html"
     r.save(fname, open_browser=False)
-    rst = scraper(block, block_vars, gallery_conf)
-    out_html = app.builder.outdir / "auto_examples" / "my_html.html"
+    out_html = outdir / "auto_examples" / "my_html.html"
     assert not out_html.is_file()
-    scraper.copyfiles()
+    rst = scraper(block, block_vars, gallery_conf)
     assert out_html.is_file()
     assert rst.count('"') == 8
     assert "<iframe" in rst

tools/circleci_bash_env.sh

@@ -4,13 +4,12 @@ set -e
 set -o pipefail
 
 ./tools/setup_xvfb.sh
-sudo apt install -qq graphviz optipng python3.10-venv python3-venv libxft2 ffmpeg
+sudo apt install -qq graphviz optipng python3.12-venv python3-venv libxft2 ffmpeg
 wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 sudo apt install ./google-chrome-stable_current_amd64.deb
-python3.10 -m venv ~/python_env
+python3.12 -m venv ~/python_env
 echo "set -e" >> $BASH_ENV
 echo "set -o pipefail" >> $BASH_ENV
-echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV
 echo "export XDG_RUNTIME_DIR=/tmp/runtime-circleci" >> $BASH_ENV
 echo "export MNE_FULL_DATE=true" >> $BASH_ENV
 source tools/get_minimal_commands.sh

tools/circleci_download.sh

@@ -2,10 +2,14 @@
 
 set -o pipefail
 export MNE_TQDM=off
+echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV
+echo "export MNE_DOC_BUILD_N_JOBS=1" >> $BASH_ENV
 
 if [ "$CIRCLE_BRANCH" == "main" ] || [[ $(cat gitlog.txt) == *"[circle full]"* ]] || [[ "$CIRCLE_BRANCH" == "maint/"* ]]; then
     echo "Doing a full build";
     echo html-memory > build.txt;
+    echo "export OPENBLAS_NUM_THREADS=1" >> $BASH_ENV
+    echo "export MNE_DOC_BUILD_N_JOBS=4" >> $BASH_ENV
     python -c "import mne; mne.datasets._download_all_example_data()";
 else
     echo "Doing a partial build";