From e8ea041ca36b74ace45fa11615a911553987a990 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Sat, 22 Jun 2024 11:58:20 -0400 Subject: [PATCH 1/4] ENH Set up parallel example execution in doc build [circle full] --- .circleci/config.yml | 4 ++-- doc/conf.py | 13 +++++-------- doc/sphinxext/mne_doc_utils.py | 19 +++++++++++++++---- mne/report/report.py | 26 +++++++++++--------------- mne/report/tests/test_report.py | 23 +++++++++++++---------- tools/circleci_bash_env.sh | 1 - tools/circleci_download.sh | 4 ++++ 7 files changed, 50 insertions(+), 40 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1826bc91569..8f85f1b66d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,8 +27,8 @@ jobs: scheduled: type: string default: "false" - docker: - - image: cimg/base:current-22.04 + machine: + image: ubuntu-2404:current # large 4 vCPUs 15GB mem # https://discuss.circleci.com/t/changes-to-remote-docker-reporting-pricing/47759 resource_class: large diff --git a/doc/conf.py b/doc/conf.py index 00f21ba18e0..09d8389565c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -15,7 +15,6 @@ from pathlib import Path import matplotlib -import pyvista import sphinx from intersphinx_registry import get_intersphinx_mapping from numpydoc import docscrape @@ -443,10 +442,6 @@ examples_dirs = ["../tutorials", "../examples"] gallery_dirs = ["auto_tutorials", "auto_examples"] os.environ["_MNE_BUILDING_DOC"] = "true" -scrapers = ("matplotlib",) -mne.viz.set_3d_backend("pyvistaqt") -pyvista.OFF_SCREEN = False -pyvista.BUILDING_GALLERY = True scrapers = ( "matplotlib", @@ -466,6 +461,7 @@ except Exception: compress_images = () +sphinx_gallery_parallel = int(os.getenv("MNE_DOC_BUILD_N_JOBS", "1")) sphinx_gallery_conf = { "doc_module": ("mne",), "reference_url": dict(mne=None), @@ -517,7 +513,7 @@ ), # called w/each script "reset_modules_order": "both", "image_scrapers": scrapers, - "show_memory": not sys.platform.startswith(("win", "darwin")), + "show_memory": sphinx_gallery_parallel == 1, "line_numbers": False, # messes with style "within_subsection_order": "FileNameSortKey", "capture_repr": ("_repr_html_",), @@ -565,6 +561,7 @@ ".*.remove.*|.*.write.*)" ), "copyfile_regex": r".*index\.rst", # allow custom index.rst files + "parallel": sphinx_gallery_parallel, } assert is_serializable(sphinx_gallery_conf) # Files were renamed from plot_* with: @@ -1650,8 +1647,8 @@ def make_version(app, exception): def setup(app): """Set up the Sphinx app.""" app.connect("autodoc-process-docstring", append_attr_meth_examples) - report_scraper.app = app - app.connect("builder-inited", report_scraper.copyfiles) + # High prio, will happen before SG + app.connect("builder-inited", report_scraper.set_dirs, priority=20) app.connect("build-finished", make_gallery_redirects) app.connect("build-finished", make_api_redirects) app.connect("build-finished", make_custom_redirects) diff --git a/doc/sphinxext/mne_doc_utils.py b/doc/sphinxext/mne_doc_utils.py index 93cfdfc1c8b..26fa7a9a393 100644 --- a/doc/sphinxext/mne_doc_utils.py +++ b/doc/sphinxext/mne_doc_utils.py @@ -6,6 +6,7 @@ import warnings import numpy as np +import pyvista import mne from mne.utils import ( @@ -86,6 +87,8 @@ def reset_warnings(gallery_conf, fname): r"open_text is deprecated\. Use files.*", # python-quantities, via neo r"numpy\.core is deprecated and has been renamed to numpy\._core", + # matplotlib + "__array_wrap__ must accept context and return_scalar.*", ): warnings.filterwarnings( # deal with other modules having bad imports "ignore", message=f".*{key}.*", category=DeprecationWarning @@ -146,6 +149,12 @@ def reset_warnings(gallery_conf, fname): "ignore", r"mne\.io\.pick.channel_indices_by_type is deprecated.*", ) + # parallel building + warnings.filterwarnings( + "ignore", + "A worker stopped while some jobs were given to the executor.*", + category=UserWarning, + ) # In case we use np.set_printoptions in any tutorials, we only # want it to affect those: @@ -159,10 +168,12 @@ def reset_modules(gallery_conf, fname, when): """Do the reset.""" import matplotlib.pyplot as plt - try: - from pyvista import Plotter # noqa - except ImportError: - Plotter = None # noqa + mne.viz.set_3d_backend("pyvistaqt") + pyvista.OFF_SCREEN = False + pyvista.BUILDING_GALLERY = True + + from pyvista import Plotter # noqa + try: from pyvistaqt import BackgroundPlotter # noqa except ImportError: diff --git a/mne/report/report.py b/mne/report/report.py index a914eac83ba..fc1676720b3 100644 --- a/mne/report/report.py +++ b/mne/report/report.py @@ -4303,10 +4303,6 @@ class _ReportScraper: is written to the same directory as the example script. """ - def __init__(self): - self.app = None - self.files = dict() - def __repr__(self): return "" @@ -4325,25 +4321,25 @@ def __call__(self, block, block_vars, gallery_conf): with open(img_fname, "w") as fid: fid.write(_FA_FILE_CODE) # copy HTML file - html_fname = op.basename(report.fname) - out_dir = op.join( - self.app.builder.outdir, - op.relpath( - op.dirname(block_vars["target_file"]), self.app.builder.srcdir - ), + html_fname = Path(report.fname).name + srcdir = Path(gallery_conf["src_dir"]) + outdir = Path(gallery_conf["out_dir"]) + out_dir = outdir / Path(block_vars["target_file"]).parent.relative_to( + srcdir ) os.makedirs(out_dir, exist_ok=True) - out_fname = op.join(out_dir, html_fname) + out_fname = out_dir / html_fname + copyfile(report.fname, out_fname) assert op.isfile(report.fname) - self.files[report.fname] = out_fname # embed links/iframe data = _SCRAPER_TEXT.format(html_fname) return data return "" - def copyfiles(self, *args, **kwargs): - for key, value in self.files.items(): - copyfile(key, value) + def set_dirs(self, app): + # Inject something into sphinx_gallery_conf as this gets pickled properly + # during parallel example generation + app.config.sphinx_gallery_conf["out_dir"] = app.builder.outdir def _df_bootstrap_table(*, df, data_id): diff --git a/mne/report/tests/test_report.py b/mne/report/tests/test_report.py index b452fee8aa2..2ed736aad8b 100644 --- a/mne/report/tests/test_report.py +++ b/mne/report/tests/test_report.py @@ -792,14 +792,18 @@ def test_scraper(tmp_path): r.add_figure(fig=fig1, title="a") r.add_figure(fig=fig2, title="b") # Mock a Sphinx + sphinx_gallery config - app = Bunch(builder=Bunch(srcdir=tmp_path, outdir=tmp_path / "_build" / "html")) + srcdir = tmp_path + outdir = tmp_path / "_build" / "html" scraper = _ReportScraper() - scraper.app = app - gallery_conf = dict(src_dir=app.builder.srcdir, builder_name="html") - img_fname = app.builder.srcdir / "auto_examples" / "images" / "sg_img.png" - target_file = app.builder.srcdir / "auto_examples" / "sg.py" + gallery_conf = dict(builder_name="html", src_dir=srcdir) + app = Bunch( + builder=Bunch(outdir=outdir), + config=Bunch(sphinx_gallery_conf=gallery_conf), + ) + scraper.set_dirs(app) + img_fname = srcdir / "auto_examples" / "images" / "sg_img.png" + target_file = srcdir / "auto_examples" / "sg.py" os.makedirs(img_fname.parent) - os.makedirs(app.builder.outdir) block_vars = dict( image_path_iterator=(img for img in [str(img_fname)]), example_globals=dict(a=1), @@ -814,12 +818,11 @@ def test_scraper(tmp_path): rst = scraper(block, block_vars, gallery_conf) # Once it's saved, add it assert rst == "" - fname = tmp_path / "my_html.html" + fname = srcdir / "my_html.html" r.save(fname, open_browser=False) - rst = scraper(block, block_vars, gallery_conf) - out_html = app.builder.outdir / "auto_examples" / "my_html.html" + out_html = outdir / "auto_examples" / "my_html.html" assert not out_html.is_file() - scraper.copyfiles() + rst = scraper(block, block_vars, gallery_conf) assert out_html.is_file() assert rst.count('"') == 8 assert "> $BASH_ENV echo "set -o pipefail" >> $BASH_ENV -echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV echo "export XDG_RUNTIME_DIR=/tmp/runtime-circleci" >> $BASH_ENV echo "export MNE_FULL_DATE=true" >> $BASH_ENV source tools/get_minimal_commands.sh diff --git a/tools/circleci_download.sh b/tools/circleci_download.sh index 2088587b1ad..44ed5193f1e 100755 --- a/tools/circleci_download.sh +++ b/tools/circleci_download.sh @@ -2,10 +2,14 @@ set -o pipefail export MNE_TQDM=off +echo "export OPENBLAS_NUM_THREADS=4" >> $BASH_ENV +echo "export MNE_DOC_BUILD_N_JOBS=1" >> $BASH_ENV if [ "$CIRCLE_BRANCH" == "main" ] || [[ $(cat gitlog.txt) == *"[circle full]"* ]] || [[ "$CIRCLE_BRANCH" == "maint/"* ]]; then echo "Doing a full build"; echo html-memory > build.txt; + echo "export OPENBLAS_NUM_THREADS=1" >> $BASH_ENV + echo "export MNE_DOC_BUILD_N_JOBS=4" >> $BASH_ENV python -c "import mne; mne.datasets._download_all_example_data()"; else echo "Doing a partial build"; From 167c29feba0888e47fdcaf6402e92ff6c87787c4 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 10 Jul 2024 15:50:13 -0400 Subject: [PATCH 2/4] FIX: Bump [circle full] --- tools/circleci_bash_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/circleci_bash_env.sh b/tools/circleci_bash_env.sh index b28fb46ed2c..151ca01383a 100755 --- a/tools/circleci_bash_env.sh +++ b/tools/circleci_bash_env.sh @@ -4,10 +4,10 @@ set -e set -o pipefail ./tools/setup_xvfb.sh -sudo apt install -qq graphviz optipng python3.10-venv python3-venv libxft2 ffmpeg +sudo apt install -qq graphviz optipng python3.12-venv python3-venv libxft2 ffmpeg wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb sudo apt install ./google-chrome-stable_current_amd64.deb -python3.10 -m venv ~/python_env +python3.12 -m venv ~/python_env echo "set -e" >> $BASH_ENV echo "set -o pipefail" >> $BASH_ENV echo "export XDG_RUNTIME_DIR=/tmp/runtime-circleci" >> $BASH_ENV From e4a64063e90c39a6055af5a727fa4c08581a870b Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 10 Jul 2024 15:55:42 -0400 Subject: [PATCH 3/4] BUG: Fix bug with choice [circle full] --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 09d8389565c..3dc6f0a00b0 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -513,7 +513,7 @@ ), # called w/each script "reset_modules_order": "both", "image_scrapers": scrapers, - "show_memory": sphinx_gallery_parallel == 1, + "show_memory": sys.platform == "linux" and sphinx_gallery_parallel == 1, "line_numbers": False, # messes with style "within_subsection_order": "FileNameSortKey", "capture_repr": ("_repr_html_",), From 9ede11dafcd8c837d1c2670caf980ee099a06fe9 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 10 Jul 2024 15:58:25 -0400 Subject: [PATCH 4/4] FIX: New cache for 3.12 [circle full] --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f85f1b66d1..1e0e38878e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,7 +111,7 @@ jobs: # Load pip cache - restore_cache: keys: - - pip-cache + - pip-cache-0 - restore_cache: keys: - user-install-bin-cache-310 @@ -123,7 +123,7 @@ jobs: ./tools/circleci_dependencies.sh - save_cache: - key: pip-cache + key: pip-cache-0 paths: - ~/.cache/pip - save_cache: @@ -422,7 +422,7 @@ jobs: command: ./tools/circleci_bash_env.sh - restore_cache: keys: - - pip-cache + - pip-cache-0 - run: name: Get Python running command: |