From 3d67e8a51747360b4dd5bad6380fad6eb65d75be Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 15:25:18 +0100 Subject: [PATCH 01/11] Run uv pip compile to pin Sphinx requirements with Python 3.11 --- requirements.in | 6 ++ requirements.txt | 265 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 requirements.in diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..e49ae8a --- /dev/null +++ b/requirements.in @@ -0,0 +1,6 @@ +Sphinx +sphinx_rtd_theme>=2.0 +sphinx_rtd_theme_ext_color_contrast +myst_nb +sphinx-lesson>=0.8.19 +pillow diff --git a/requirements.txt b/requirements.txt index 135f7aa..6e084f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,260 @@ -Sphinx -sphinx_rtd_theme -sphinx_rtd_theme_ext_color_contrast -myst_nb -sphinx-lesson +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt +alabaster==0.7.16 + # via sphinx +anyio==4.8.0 + # via + # starlette + # watchfiles +asttokens==3.0.0 + # via stack-data +attrs==24.3.0 + # via + # jsonschema + # jupyter-cache + # referencing +babel==2.16.0 + # via sphinx +certifi==2024.12.14 + # via requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via + # jupyter-cache + # uvicorn +colorama==0.4.6 + # via sphinx-autobuild +comm==0.2.2 + # via ipykernel +debugpy==1.8.11 + # via ipykernel +decorator==5.1.1 + # via ipython +docutils==0.21.2 + # via + # myst-parser + # sphinx + # sphinx-rtd-theme + # sphinx-tabs + # sphinx-togglebutton +executing==2.1.0 + # via stack-data +fastjsonschema==2.21.1 + # via nbformat +greenlet==3.1.1 + # via sqlalchemy +h11==0.14.0 + # via uvicorn +idna==3.10 + # via + # anyio + # requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==8.5.0 + # via + # jupyter-cache + # myst-nb +ipykernel==6.29.5 + # via myst-nb +ipython==8.31.0 + # via + # ipykernel + # myst-nb +jedi==0.19.2 + # via ipython +jinja2==3.1.5 + # via + # myst-parser + # sphinx +jsonschema==4.23.0 + # via nbformat +jsonschema-specifications==2024.10.1 + # via jsonschema +jupyter-cache==1.0.1 + # via myst-nb +jupyter-client==8.6.3 + # via + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbformat +markdown-it-py==3.0.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==3.0.2 + # via jinja2 +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdit-py-plugins==0.4.2 + # via myst-parser +mdurl==0.1.2 + # via markdown-it-py +myst-nb==1.1.2 + # via + # -r requirements.in + # sphinx-lesson +myst-parser==4.0.0 + # via myst-nb +nbclient==0.10.2 + # via + # jupyter-cache + # myst-nb +nbformat==5.10.4 + # via + # jupyter-cache + # myst-nb + # nbclient +nest-asyncio==1.6.0 + # via ipykernel +packaging==24.2 + # via + # ipykernel + # sphinx +parso==0.8.4 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==11.1.0 + # via -r requirements.in +platformdirs==4.3.6 + # via jupyter-core +prompt-toolkit==3.0.48 + # via ipython +psutil==6.1.1 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pygments==2.19.1 + # via + # ipython + # sphinx + # sphinx-tabs +python-dateutil==2.9.0.post0 + # via jupyter-client +pyyaml==6.0.2 + # via + # jupyter-cache + # myst-nb + # myst-parser +pyzmq==26.2.0 + # via + # ipykernel + # jupyter-client +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via sphinx +rpds-py==0.22.3 + # via + # jsonschema + # referencing +setuptools==75.7.0 + # via sphinx-togglebutton +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.4.7 + # via + # -r requirements.in + # myst-nb + # myst-parser + # sphinx-autobuild + # sphinx-copybutton + # sphinx-lesson + # sphinx-rtd-theme + # sphinx-rtd-theme-ext-color-contrast + # sphinx-tabs + # sphinx-togglebutton + # sphinxcontrib-jquery +sphinx-autobuild==2024.10.3 + # via sphinx-lesson +sphinx-copybutton==0.5.2 + # via sphinx-lesson +sphinx-lesson==0.8.19 + # via -r requirements.in +sphinx-minipres==0.2.1 + # via sphinx-lesson +sphinx-rtd-theme==3.0.2 + # via + # -r requirements.in + # sphinx-lesson +sphinx-rtd-theme-ext-color-contrast==0.3.2 + # via + # -r requirements.in + # sphinx-lesson +sphinx-tabs==3.4.7 + # via sphinx-lesson +sphinx-togglebutton==0.3.2 + # via sphinx-lesson +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +sqlalchemy==2.0.36 + # via jupyter-cache +stack-data==0.6.3 + # via ipython +starlette==0.45.2 + # via sphinx-autobuild +tabulate==0.9.0 + # via jupyter-cache +tornado==6.4.2 + # via + # ipykernel + # jupyter-client +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbformat +typing-extensions==4.12.2 + # via + # anyio + # ipython + # myst-nb + # sqlalchemy +urllib3==2.3.0 + # via requests +uvicorn==0.34.0 + # via sphinx-autobuild +watchfiles==1.0.3 + # via sphinx-autobuild +wcwidth==0.2.13 + # via prompt-toolkit +websockets==14.1 + # via sphinx-autobuild +wheel==0.45.1 + # via sphinx-togglebutton +zipp==3.21.0 + # via importlib-metadata From 1a4f44d00c6ed2f500700e43dad2ae96ee39c2bc Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 15:25:31 +0100 Subject: [PATCH 02/11] Add livehtml make rule --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index d50f54e..46f35b9 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,7 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Live reload site documents for local development +livehtml: + sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) From 56f0afc5a8b676aaab3cc192234bcd9ef354f902 Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 15:58:52 +0100 Subject: [PATCH 03/11] Move eurohpc instruction to optional material --- content/index.rst | 3 +- content/setup-eurohpc.rst | 246 ++++++++++++++++++++++++++++++++++++++ content/setup.rst | 241 ------------------------------------- 3 files changed, 247 insertions(+), 243 deletions(-) create mode 100644 content/setup-eurohpc.rst diff --git a/content/index.rst b/content/index.rst index a6322ee..a634207 100644 --- a/content/index.rst +++ b/content/index.rst @@ -62,13 +62,12 @@ and distributed computing. performance-boosting dask - - .. toctree:: :maxdepth: 1 :caption: Optional material + setup-eurohpc pandas-extra GPU-computing diff --git a/content/setup-eurohpc.rst b/content/setup-eurohpc.rst new file mode 100644 index 0000000..5a96d10 --- /dev/null +++ b/content/setup-eurohpc.rst @@ -0,0 +1,246 @@ +EuroHPC systems +--------------- + +.. warning:: + + These instructions may be outdated and were last updated in 2023. + + +Here are instructions for accessing the EuroHPC system, setting up the Python environment +and running jobs. Please follow the instructions for the HPC system that will be used +during the workshop that you are attending. + +.. tabs:: + + .. group-tab:: Vega + + Thanks to `IZUM `__ in Slovenia we will have an allocation on the + petascale `Vega `__ EuroHPC system for the duration of the workshop. + The sustained and peak performance of Vega is 6.9 petaflops and 10.1 petaflops, respectively. + + **Architecture**: + + Vega has both `GPU and CPU partititions `__: + + - CPU partition: Each node has two AMD Epyc 7H12 CPUs, each with 64 cores. 768 nodes with 256 GB, + 192 nodes with 1 TB of RAM DDR4-3200, local 1.92 TB M.2 SSD. + - GPU partition: Each node has 4 GPUs NVidia A100 with 40 GB HBMI2 and two AMD Epyc 7H12 CPUs. + In total 60 nodes with 512 GB of RAM DDR4-3200, local 1.92 TB M.2 SSD + + .. group-tab:: Karolina + + Thanks to `IT4I `__ in the Czech Republic we will have an allocation + on the petascale `Karolina supercomputer `__ + for the duration of the workshop. The peak performance of Karolina is 15.7 petaflops. + + **Architecture**: + + - 720x 2x AMD 7H12, 64 cores, 2,6 GHz, 92,160 cores in total + - 72x 2x AMD 7763, 64 cores, 2,45 GHz, 9,216 cores in total + - 72x 8x NVIDIA A100 GPU, 576 GPU in total + - 32x Intel Xeon-SC 8628, 24 cores, 2,9 GHz, 768 cores in total + - 36x 2x AMD 7H12, 64 cores, 2,6 GHz, 4,608 cores in total + - 2x 2x AMD 7452, 32 cores, 2,35 GHz, 128 cores in total + +Software on the cluster is available through a module system. +First load the Anaconda module to get access to the ``conda`` package manager: + +.. tabs:: + + .. group-tab:: Vega + + .. code-block:: console + + $ #check available Anaconda modules: + $ ml av Anaconda3 + $ ml add Anaconda3/2020.11 + + .. group-tab:: Karolina + + .. code-block:: console + + $ #check available Anaconda modules: + $ ml av Anaconda3 + $ ml add Anaconda3/2021.11 + + +To be able to create conda environments in your home directory you need to initialize it. +The following command adds the necessary configuration to your ``.bashrc`` file: + +.. code-block:: console + + $ conda init bash + +You now need to either log in to the cluster again or start a new shell session by typing ``bash``: + +.. code-block:: console + + $ bash + +Now, either create a new environment with all required dependencies or activate +a pre-existing environment created in a directory you have access to: + +.. tabs:: + + .. tab:: Create new environment in $HOME + + .. code-block:: console + + $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml + + The installation can take several minutes. + Now activate the environment by: + + .. code-block:: console + + $ conda activate pyhpda + + .. tab:: Activate existing environment + + .. code-block:: console + + $ conda activate /path/to/envdir/ + + +mpi4py +^^^^^^ + +Additional steps are required to use mpi4py since the Python package needs to be +linked with the system's MPI libraries. + +.. tabs:: + + .. group-tab:: Vega + + To use mpi4py you need to load a module which contains MPI libraries and then install ``mpi4py`` + using ``pip``: + + .. code-block:: console + + $ ml add foss/2020b + $ CC=gcc MPICC=mpicc python3 -m pip install mpi4py --no-binary=mpi4py + + .. group-tab:: Karolina + + To use mpi4py you only need to load a module: + + .. code-block:: console + + $ ml add mpi4py/3.1.1-gompi-2020b + +Running jobs +^^^^^^^^^^^^ + +Resources can be allocated both through batch jobs (submitting a script to the scheduler) +and interactively. You will need to provide a project ID when asking for an allocation. +To find out what projects you belong to on the cluster, type: + +.. code-block:: console + + $ sacctmgr -p show associations user=$USER + +The second column of the output contains the project ID. + +.. tabs:: + + .. group-tab:: Vega + + Vega uses the SLURM scheduler. + Use the following command to allocate one interactive node with 8 cores for 1 hour + in the CPU partition. If there is a reservation on the cluster for the workshop, + add ``--reservation=RESERVATIONNAME`` to the command. + + .. code-block:: console + + $ salloc -N 1 --ntasks-per-node=8 --ntasks-per-core=1 -A --partition=cpu -t 01:00:00 + + To instead book a GPU node, type (again adding reservation flag if relevant): + + .. code-block:: console + + $ salloc -N 1 --ntasks-per-node=1 --ntasks-per-core=1 -A --partition=gpu --gres=gpu:1 --cpus-per-task 1 -t 01:00:00 + + .. group-tab:: Karolina + + Karolina uses the PBS scheduler. + To allocate one interactive node for + 1 hour on 1 node in the CPU partition and express queue: + + .. code-block:: console + + $ qsub -A DD-22-28 -q qexp -l walltime=01:00:00 -I + +Running Jupyter +^^^^^^^^^^^^^^^ + +The following procedure starts a Jupyter-Lab server on a compute node, creates an SSH tunnel from +your local machine to the compute node, and then connects to the remote Jupyter-Lab server from your +browser. + +First make sure to follow the above instructions to: + +- Allocate an interactive compute node for a sufficiently long time +- Switch to the pyhpda conda environment. + +After allocating an interactive node you will see the name of the node in the output. + +.. tabs:: + + .. group-tab:: Vega + + After allocating an interactive node you will see the name of the node in the + output, e.g. ``salloc: Nodes cn0709 are ready for job``. + + You now need to ssh to that node, switch to the pyhpda + conda environment, and start the Jupyter-Lab server on a particular port + (choose one between 8000 and 9000) + and IP address (the name of the compute node). Also load a module containing + OpenMPI to have access to MPI inside Jupyter: + + .. code-block:: console + + $ ssh cn0709 + $ conda activate pyhpda + $ ml add foss/2021b + $ jupyter-lab --no-browser --port=8123 --ip=cn0709 + + .. group-tab:: Karolina + + After allocating an interactive node your terminal session will be connected to that node. + Find out the name of your compute node. Your terminal prompt should show it but you can + also run the hostname command. Look only at the node name (e.g. cn012) and disregard + the ``.karolina.it4i.cz`` part. + + Now start the Jupyter-Lab server on a particular port + (choose one between 8000 and 9000) + and IP address (the name of the compute node): + + .. code-block:: console + + $ jupyter-lab --no-browser --port=8123 --ip=cn012 + + +Now create an SSH tunnel **from a new terminal on your local machine** to the correct +port and IP: + +.. tabs:: + + .. group-tab:: Vega + + .. code-block:: console + + $ ssh -TN -f YourUsername@login.vega.izum.si -L localhost:8123:cn0709:8123 -L localhost:8787:cn0709:8787 + + .. group-tab:: Karolina + + .. code-block:: console + + $ ssh -TN -f YourUsername@login2.karolina.it4i.cz -L localhost:8123:cn012:8123 + +Go back to the terminal running Jupyter-Lab on the compute node, and copy-paste the URL +starting with ``127.0.0.1`` which contains a long token into your local browser. +If that does not work, try replacing ``127.0.0.1`` with ``localhost``. + +If everything is working as it should, you should now be able to create a new Jupyter notebook in your browser +which is connected to the compute node and the ``pyhpda`` conda environment. + diff --git a/content/setup.rst b/content/setup.rst index 35f079c..3393d6f 100644 --- a/content/setup.rst +++ b/content/setup.rst @@ -52,244 +52,3 @@ Finally, open Jupyter-Lab in your browser: $ jupyter-lab -EuroHPC systems ---------------- - -Here are instructions for accessing the EuroHPC system, setting up the Python environment -and running jobs. Please follow the instructions for the HPC system that will be used -during the workshop that you are attending. - -.. tabs:: - - .. group-tab:: Vega - - Thanks to `IZUM `__ in Slovenia we will have an allocation on the - petascale `Vega `__ EuroHPC system for the duration of the workshop. - The sustained and peak performance of Vega is 6.9 petaflops and 10.1 petaflops, respectively. - - **Architecture**: - - Vega has both `GPU and CPU partititions `__: - - - CPU partition: Each node has two AMD Epyc 7H12 CPUs, each with 64 cores. 768 nodes with 256 GB, - 192 nodes with 1 TB of RAM DDR4-3200, local 1.92 TB M.2 SSD. - - GPU partition: Each node has 4 GPUs NVidia A100 with 40 GB HBMI2 and two AMD Epyc 7H12 CPUs. - In total 60 nodes with 512 GB of RAM DDR4-3200, local 1.92 TB M.2 SSD - - .. group-tab:: Karolina - - Thanks to `IT4I `__ in the Czech Republic we will have an allocation - on the petascale `Karolina supercomputer `__ - for the duration of the workshop. The peak performance of Karolina is 15.7 petaflops. - - **Architecture**: - - - 720x 2x AMD 7H12, 64 cores, 2,6 GHz, 92,160 cores in total - - 72x 2x AMD 7763, 64 cores, 2,45 GHz, 9,216 cores in total - - 72x 8x NVIDIA A100 GPU, 576 GPU in total - - 32x Intel Xeon-SC 8628, 24 cores, 2,9 GHz, 768 cores in total - - 36x 2x AMD 7H12, 64 cores, 2,6 GHz, 4,608 cores in total - - 2x 2x AMD 7452, 32 cores, 2,35 GHz, 128 cores in total - -Software on the cluster is available through a module system. -First load the Anaconda module to get access to the ``conda`` package manager: - -.. tabs:: - - .. group-tab:: Vega - - .. code-block:: console - - $ #check available Anaconda modules: - $ ml av Anaconda3 - $ ml add Anaconda3/2020.11 - - .. group-tab:: Karolina - - .. code-block:: console - - $ #check available Anaconda modules: - $ ml av Anaconda3 - $ ml add Anaconda3/2021.11 - - -To be able to create conda environments in your home directory you need to initialize it. -The following command adds the necessary configuration to your ``.bashrc`` file: - -.. code-block:: console - - $ conda init bash - -You now need to either log in to the cluster again or start a new shell session by typing ``bash``: - -.. code-block:: console - - $ bash - -Now, either create a new environment with all required dependencies or activate -a pre-existing environment created in a directory you have access to: - -.. tabs:: - - .. tab:: Create new environment in $HOME - - .. code-block:: console - - $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml - - The installation can take several minutes. - Now activate the environment by: - - .. code-block:: console - - $ conda activate pyhpda - - .. tab:: Activate existing environment - - .. code-block:: console - - $ conda activate /path/to/envdir/ - - -mpi4py -^^^^^^ - -Additional steps are required to use mpi4py since the Python package needs to be -linked with the system's MPI libraries. - -.. tabs:: - - .. group-tab:: Vega - - To use mpi4py you need to load a module which contains MPI libraries and then install ``mpi4py`` - using ``pip``: - - .. code-block:: console - - $ ml add foss/2020b - $ CC=gcc MPICC=mpicc python3 -m pip install mpi4py --no-binary=mpi4py - - .. group-tab:: Karolina - - To use mpi4py you only need to load a module: - - .. code-block:: console - - $ ml add mpi4py/3.1.1-gompi-2020b - -Running jobs -^^^^^^^^^^^^ - -Resources can be allocated both through batch jobs (submitting a script to the scheduler) -and interactively. You will need to provide a project ID when asking for an allocation. -To find out what projects you belong to on the cluster, type: - -.. code-block:: console - - $ sacctmgr -p show associations user=$USER - -The second column of the output contains the project ID. - -.. tabs:: - - .. group-tab:: Vega - - Vega uses the SLURM scheduler. - Use the following command to allocate one interactive node with 8 cores for 1 hour - in the CPU partition. If there is a reservation on the cluster for the workshop, - add ``--reservation=RESERVATIONNAME`` to the command. - - .. code-block:: console - - $ salloc -N 1 --ntasks-per-node=8 --ntasks-per-core=1 -A --partition=cpu -t 01:00:00 - - To instead book a GPU node, type (again adding reservation flag if relevant): - - .. code-block:: console - - $ salloc -N 1 --ntasks-per-node=1 --ntasks-per-core=1 -A --partition=gpu --gres=gpu:1 --cpus-per-task 1 -t 01:00:00 - - .. group-tab:: Karolina - - Karolina uses the PBS scheduler. - To allocate one interactive node for - 1 hour on 1 node in the CPU partition and express queue: - - .. code-block:: console - - $ qsub -A DD-22-28 -q qexp -l walltime=01:00:00 -I - -Running Jupyter -^^^^^^^^^^^^^^^ - -The following procedure starts a Jupyter-Lab server on a compute node, creates an SSH tunnel from -your local machine to the compute node, and then connects to the remote Jupyter-Lab server from your -browser. - -First make sure to follow the above instructions to: - -- Allocate an interactive compute node for a sufficiently long time -- Switch to the pyhpda conda environment. - -After allocating an interactive node you will see the name of the node in the output. - -.. tabs:: - - .. group-tab:: Vega - - After allocating an interactive node you will see the name of the node in the - output, e.g. ``salloc: Nodes cn0709 are ready for job``. - - You now need to ssh to that node, switch to the pyhpda - conda environment, and start the Jupyter-Lab server on a particular port - (choose one between 8000 and 9000) - and IP address (the name of the compute node). Also load a module containing - OpenMPI to have access to MPI inside Jupyter: - - .. code-block:: console - - $ ssh cn0709 - $ conda activate pyhpda - $ ml add foss/2021b - $ jupyter-lab --no-browser --port=8123 --ip=cn0709 - - .. group-tab:: Karolina - - After allocating an interactive node your terminal session will be connected to that node. - Find out the name of your compute node. Your terminal prompt should show it but you can - also run the hostname command. Look only at the node name (e.g. cn012) and disregard - the ``.karolina.it4i.cz`` part. - - Now start the Jupyter-Lab server on a particular port - (choose one between 8000 and 9000) - and IP address (the name of the compute node): - - .. code-block:: console - - $ jupyter-lab --no-browser --port=8123 --ip=cn012 - - -Now create an SSH tunnel **from a new terminal on your local machine** to the correct -port and IP: - -.. tabs:: - - .. group-tab:: Vega - - .. code-block:: console - - $ ssh -TN -f YourUsername@login.vega.izum.si -L localhost:8123:cn0709:8123 -L localhost:8787:cn0709:8787 - - .. group-tab:: Karolina - - .. code-block:: console - - $ ssh -TN -f YourUsername@login2.karolina.it4i.cz -L localhost:8123:cn012:8123 - -Go back to the terminal running Jupyter-Lab on the compute node, and copy-paste the URL -starting with ``127.0.0.1`` which contains a long token into your local browser. -If that does not work, try replacing ``127.0.0.1`` with ``localhost``. - -If everything is working as it should, you should now be able to create a new Jupyter notebook in your browser -which is connected to the compute node and the ``pyhpda`` conda environment. - From c5307a5b3cd5cc3a6971ddd38204b1a0607b0ab7 Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:02:02 +0100 Subject: [PATCH 04/11] Add optional parallel computing to index --- content/index.rst | 1 + content/parallel-computing_opt.rst | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content/index.rst b/content/index.rst index a634207..f591db4 100644 --- a/content/index.rst +++ b/content/index.rst @@ -70,6 +70,7 @@ and distributed computing. setup-eurohpc pandas-extra GPU-computing + parallel-computing_opt .. toctree:: diff --git a/content/parallel-computing_opt.rst b/content/parallel-computing_opt.rst index fe2beb6..39167e4 100644 --- a/content/parallel-computing_opt.rst +++ b/content/parallel-computing_opt.rst @@ -1,5 +1,7 @@ - # on some HPC systems you might need 'srun -n 4' instead of 'mpirun -np 4' - # on Vega, add this module for MPI libraries: ml add foss/2020b +.. note:: + + on some HPC systems you might need ``srun -n 4`` instead of ``mpirun -np 4`` + on Vega, add this module for MPI libraries: ``ml add foss/2020b`` .. callout:: MPI libraries From 9998c47394418965ddecd6d5246d96b6ae20904a Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:03:04 +0100 Subject: [PATCH 05/11] Add mpi4py and rm snakemake-minimal from env --- content/env/environment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/content/env/environment.yml b/content/env/environment.yml index 703afb0..ea2eb50 100644 --- a/content/env/environment.yml +++ b/content/env/environment.yml @@ -1,8 +1,6 @@ name: pyhpda channels: - conda-forge - - defaults - - bioconda dependencies: - numpy - scipy @@ -15,13 +13,13 @@ dependencies: - netcdf4 - matplotlib - dask + - mpi4py - graphviz - python-graphviz - ipywidgets - widgetsnbextension - dask-jobqueue - dask-labextension - - snakemake-minimal - xarray - jupyter - jupyterlab From 7d621a398f276ce09fa6a0b70eb7e320ccb38e3d Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:22:09 +0100 Subject: [PATCH 06/11] Sync with sphinx-lesson-template --- LICENSE.code | 3 +-- content/_static/overrides.css | 16 +++++++-------- content/conf.py | 38 ++++++++++++++++++++++++++--------- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/LICENSE.code b/LICENSE.code index 3f02770..6393229 100644 --- a/LICENSE.code +++ b/LICENSE.code @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022, HPDA-Python and individual contributors. +Copyright (c) 2024, ENCCS and individual contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/content/_static/overrides.css b/content/_static/overrides.css index 26a0f16..56413f4 100644 --- a/content/_static/overrides.css +++ b/content/_static/overrides.css @@ -26,7 +26,7 @@ background: #DDDDDD; } .rst-content .instructor-note > .admonition-title { - background: #BBBBBB; + background: #595959; } .rst-content .instructor-note > .admonition-title::before { content: ""; @@ -37,7 +37,7 @@ background: #EEEEBB; } .rst-content .callout > .admonition-title { - background: #BBCC33; + background: #595959; } /* questions */ @@ -45,15 +45,15 @@ background: rgba(253, 219, 199, 0.3); } .rst-content .questions > .admonition-title { - background: rgba(204, 51, 17, 0.5); + background: #993360; } /* discussion */ .rst-content .discussion { - background: rgba(231, 212, 232 0.3); + background: rgba(231, 212, 232, 0.3); } .rst-content .discussion > .admonition-title { - background: rgba(194, 165, 207, 0.5); + background: #714884; } /* signature */ @@ -61,7 +61,7 @@ background: rgba(217, 240, 211, 0.3); } .rst-content .signature > .admonition-title { - background: rgba(172, 211, 158, 0.5); + background: #21634b; } .rst-content .signature > .admonition-title::before { content: "\01F527"; @@ -72,7 +72,7 @@ background: rgba(217, 240, 211, 0.0); } .rst-content .parameters > .admonition-title { - background: rgba(172, 211, 158, 0.5); + background: #21634b; } .rst-content .parameters > .admonition-title::before { content: "\01F4BB"; @@ -83,7 +83,7 @@ background: rgba(221, 221, 221, 0.3); } .rst-content .typealong > .admonition-title { - background: rgba(187, 187, 187, 1.0); + background: rgba(89, 89, 89, 1.0); } .rst-content .typealong > .admonition-title::before { content: "\02328"; diff --git a/content/conf.py b/content/conf.py index 54867ca..b3452fd 100644 --- a/content/conf.py +++ b/content/conf.py @@ -21,7 +21,7 @@ copyright = "2022, ENCCS and individual contributors." author = "ENCCS and individual contributors." github_user = "ENCCS" -github_repo_name = "HPDA-Python" +github_repo_name = "" # auto-detected from dirname if blank github_version = "main" conf_py_path = "/content/" # with leading and trailing slash @@ -44,7 +44,15 @@ # jupyter_execute_notebooks = "off" # jupyter_execute_notebooks = "auto" # *only* execute if at least one output is missing. # jupyter_execute_notebooks = "force" -jupyter_execute_notebooks = "cache" +nb_execution_mode = "cache" + +# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html +myst_enable_extensions = [ + "colon_fence", +] + +# Settings for sphinx-copybutton +copybutton_exclude = ".linenos, .gp" # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] @@ -73,7 +81,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = ["overrides.css"] # HTML context: @@ -84,7 +92,7 @@ "github_user": github_user, # Auto-detect directory name. This can break, but # useful as a default. - "github_repo": github_repo_name or basename(dirname(realpath(__file__))), + "github_repo": github_repo_name or basename(dirname(dirname(realpath(__file__)))), "github_version": github_version, "conf_py_path": conf_py_path, } @@ -120,17 +128,27 @@ class TypealongDirective(_BaseCRDirective): extra_classes = ["toggle-shown", "dropdown"] -DIRECTIVES = [SignatureDirective, ParametersDirective, TypealongDirective] +DIRECTIVES: list[type[_BaseCRDirective]] = [ + SignatureDirective, + ParametersDirective, + TypealongDirective, +] def setup(app): for obj in DIRECTIVES: - app.add_directive(obj.get_cssname(), obj) + app.add_directive(obj.cssname(), obj) + import os -if os.environ.get('GITHUB_REF', '') == 'refs/heads/main': + +if os.environ.get("GITHUB_REF", "") == "refs/heads/main": html_js_files = [ - ('https://plausible.io/js/script.js', {"data-domain": "enccs.github.io/hpda-python", "defer": "\ -defer"}), + ( + "https://plausible.io/js/script.js", + { + "data-domain": "enccs.github.io/hpda-python", + "defer": "defer", + }, + ), ] - From e32b62ff2e91117162eea8daaa0d871137b6ae3e Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:22:25 +0100 Subject: [PATCH 07/11] Update title of optional content --- content/parallel-computing_opt.rst | 3 +++ content/setup-eurohpc.rst | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content/parallel-computing_opt.rst b/content/parallel-computing_opt.rst index 39167e4..a73de7d 100644 --- a/content/parallel-computing_opt.rst +++ b/content/parallel-computing_opt.rst @@ -1,3 +1,6 @@ +More on parallel computing +========================== + .. note:: on some HPC systems you might need ``srun -n 4`` instead of ``mpirun -np 4`` diff --git a/content/setup-eurohpc.rst b/content/setup-eurohpc.rst index 5a96d10..4182d33 100644 --- a/content/setup-eurohpc.rst +++ b/content/setup-eurohpc.rst @@ -1,5 +1,5 @@ -EuroHPC systems ---------------- +Installation in EuroHPC systems +------------------------------- .. warning:: From debdb5c0680215f343174db498beccb406f000ef Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:50:39 +0100 Subject: [PATCH 08/11] Switch to Miniforge --- .gitignore | 4 ++++ content/setup.rst | 43 +++++++++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index c0d3369..078e14b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ /venv* .jupyter_cache jupyter_execute + +# pixi environments +.pixi +*.egg-info diff --git a/content/setup.rst b/content/setup.rst index 3393d6f..a575297 100644 --- a/content/setup.rst +++ b/content/setup.rst @@ -2,29 +2,27 @@ Installation and HPC access =========================== This page contains instructions for installing the required dependencies on a local computer -as well as instructions for logging in and using two EuroHPC systems. +as well as instructions for logging into a EuroHPC system. Local installation ------------------ -If you already have a preferred way to manage Python versions and -libraries, you can stick to that. If not, we recommend that you -install Python3 and all libraries using -`miniconda `__, -a free minimal installer for the package, dependency and environment manager +If you already have a preferred way to manage Python versions and +libraries, you can stick to that. If not, we recommend that you install +Python3 and all libraries using +`Miniforge `__, a free +minimal installer for the package, dependency and environment manager `conda `__. -Please follow the installation instructions on -https://docs.conda.io/en/latest/miniconda.html to install Miniconda3. +Please follow the installation instructions on +https://conda-forge.org/download/ to install Miniforge. -Make sure that both Python and conda are correctly installed: +Make sure that conda is correctly installed: .. code-block:: console - $ python --version - $ # should give something like Python 3.9.7 $ conda --version - $ # should give something like conda 4.10.2 + $ # should give something like conda 24.11.2 With conda installed, install the required dependencies by running: @@ -38,12 +36,12 @@ This will create a new environment ``pyhpda`` which you need to activate by: $ conda activate pyhpda -To use MPI4Py on your computer you need to install MPI libraries. With conda, these libraries are -installed automatically when installing the mpi4py package: - -.. code-block:: console - - $ conda install -c conda-forge mpi4py +.. To use MPI4Py on your computer you need to install MPI libraries. With conda, these libraries are +.. installed automatically when installing the mpi4py package: +.. +.. .. code-block:: console +.. +.. $ conda install -c conda-forge mpi4py Finally, open Jupyter-Lab in your browser: @@ -52,3 +50,12 @@ Finally, open Jupyter-Lab in your browser: $ jupyter-lab +LUMI +------------ + +.. note:: + + .. todo:: + Add Instructions for using this + + Go to LUMI `open OnDemand portal `__ From ae6d375ae50273df1edb23df46756281d2e327b6 Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:58:59 +0100 Subject: [PATCH 09/11] Add scalene --- content/env/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/content/env/environment.yml b/content/env/environment.yml index ea2eb50..33d4c95 100644 --- a/content/env/environment.yml +++ b/content/env/environment.yml @@ -14,6 +14,7 @@ dependencies: - matplotlib - dask - mpi4py + - scalene - graphviz - python-graphviz - ipywidgets From f498fd58e9a6fc593a471446a59590e194a750e3 Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 16:59:14 +0100 Subject: [PATCH 10/11] Check for Python version --- content/setup.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/content/setup.rst b/content/setup.rst index a575297..6051e3e 100644 --- a/content/setup.rst +++ b/content/setup.rst @@ -22,13 +22,13 @@ Make sure that conda is correctly installed: .. code-block:: console $ conda --version - $ # should give something like conda 24.11.2 + conda 24.11.2 With conda installed, install the required dependencies by running: .. code-block:: console - $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml + $ conda env create --yes -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml This will create a new environment ``pyhpda`` which you need to activate by: @@ -43,6 +43,13 @@ This will create a new environment ``pyhpda`` which you need to activate by: .. .. $ conda install -c conda-forge mpi4py +Ensure that the Python version is fairly recent: + +.. code-block:: console + + $ python --version + Python 3.12.8 + Finally, open Jupyter-Lab in your browser: .. code-block:: console From 6af06a38d812fe8414f5298e1dde8ec228f3ec5e Mon Sep 17 00:00:00 2001 From: "Ashwin V. Mohanan" Date: Wed, 8 Jan 2025 17:02:51 +0100 Subject: [PATCH 11/11] Update GH actions --- .github/workflows/sphinx.yml | 180 ++++++++++++++++++++--------------- 1 file changed, 101 insertions(+), 79 deletions(-) diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 8d16ada..24247ce 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -1,54 +1,69 @@ -# From: https://github.com/rkdarst/sphinx-actions-test/blob/master/.github/workflows/sphinx-build.yml +# Deploy Sphinx. This could be shorter, but we also do some extra +# stuff. +# +# License: CC-0. This is the canonical location of this file, which +# you may want to link to anyway: +# https://github.com/coderefinery/sphinx-lesson-template/blob/main/.github/workflows/sphinx.yml +# https://raw.githubusercontent.com/coderefinery/sphinx-lesson-template/main/.github/workflows/sphinx.yml + name: sphinx on: [push, pull_request] -# If these SPHINXOPTS are enabled, then be strict about the builds and -# fail on any warnings -#env: -# SPHINXOPTS: "-W --keep-going -T" env: - DEFAULT_BRANCH: main + DEFAULT_BRANCH: "main" + # If these SPHINXOPTS are enabled, then be strict about the + # builds and fail on any warnings. + #SPHINXOPTS: "-W --keep-going -T" + GENERATE_PDF: true # to enable, must be 'true' lowercase + GENERATE_SINGLEHTML: true # to enable, must be 'true' lowercase + PDF_FILENAME: lesson.pdf + MULTIBRANCH: true # to enable, must be 'true' lowercase jobs: - build-and-deploy: - name: Build and gh-pages + build: + name: Build runs-on: ubuntu-latest + permissions: + contents: read + steps: # https://github.com/marketplace/actions/checkout - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 lfs: true + # https://github.com/marketplace/actions/setup-python # ^-- This gives info on matrix testing. - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - # https://docs.github.com/en/actions/guides/building-and-testing-python#caching-dependencies - # ^-- How to set up caching for pip on Ubuntu - - name: Cache pip - uses: actions/cache@v2 + uses: actions/setup-python@v4 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + python-version: '3.11' + cache: 'pip' + # https://docs.github.com/en/actions/guides/building-and-testing-python#installing-dependencies # ^-- This gives info on installing dependencies with pip - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt + + # Debug - name: Debugging information + env: + ref: ${{github.ref}} + event_name: ${{github.event_name}} + head_ref: ${{github.head_ref}} + base_ref: ${{github.base_ref}} run: | - echo "github.ref:" ${{github.ref}} - echo "github.event_name:" ${{github.event_name}} - echo "github.head_ref:" ${{github.head_ref}} - echo "github.base_ref:" ${{github.base_ref}} + echo "github.ref: ${ref}" + echo "github.event_name: ${event_name}" + echo "github.head_ref: ${head_ref}" + echo "github.base_ref: ${base_ref}" + echo "GENERATE_PDF: ${GENERATE_PDF}" + echo "GENERATE_SINGLEHTML: ${GENERATE_SINGLEHTML}" set -x git rev-parse --abbrev-ref HEAD git branch @@ -61,7 +76,8 @@ jobs: # Build - uses: ammaraskar/sphinx-problem-matcher@master - - name: Build Sphinx docs + - name: Build Sphinx docs (dirhtml) + # SPHINXOPTS used via environment variables run: | make dirhtml # This fixes broken copy button icons, as explained in @@ -73,60 +89,38 @@ jobs: # https://github.com/readthedocs/sphinx_rtd_theme/pull/1025 sed -i 's/url_root="#"/url_root=""/' _build/dirhtml/index.html || true + # singlehtml + - name: Generate singlehtml + if: ${{ env.GENERATE_SINGLEHTML == 'true' }} + run: | + make singlehtml + mv _build/singlehtml/ _build/dirhtml/singlehtml/ - # The following supports building all branches and combining on - # gh-pages + # PDF if requested + - name: Generate PDF + if: ${{ env.GENERATE_PDF == 'true' }} + run: | + pip install https://github.com/rkdarst/sphinx_pyppeteer_builder/archive/refs/heads/main.zip + make pyppeteer + mv _build/pyppeteer/*.pdf _build/dirhtml/${PDF_FILENAME} - # Clone and set up the old gh-pages branch - - name: Clone old gh-pages + # Stage all deployed assets in _gh-pages/ for simplicity, and to + # prepare to do a multi-branch deployment. + - name: Copy deployment data to _gh-pages/ if: ${{ github.event_name == 'push' }} - run: | - set -x - git fetch - ( git branch gh-pages remotes/origin/gh-pages && git clone . --branch=gh-pages _gh-pages/ ) || mkdir _gh-pages - rm -rf _gh-pages/.git/ - mkdir -p _gh-pages/branch/ - # If a push and default branch, copy build to _gh-pages/ as the "main" - # deployment. - - name: Copy new build (default branch) - if: | - contains(github.event_name, 'push') && - contains(github.ref, env.DEFAULT_BRANCH) - run: | - set -x - # Delete everything under _gh-pages/ that is from the - # primary branch deployment. Eicludes the other branches - # _gh-pages/branch-* paths, and not including - # _gh-pages itself. - find _gh-pages/ -mindepth 1 ! -path '_gh-pages/branch*' -delete + run: rsync -a _build/dirhtml/ _gh-pages/ - # If a push and not on default branch, then copy the build to - # _gh-pages/branch/$brname (transforming '/' into '--') - - name: Copy new build (branch) - if: | - contains(github.event_name, 'push') && - !contains(github.ref, env.DEFAULT_BRANCH) - run: | - set -x - #brname=$(git rev-parse --abbrev-ref HEAD) - brname="${{github.ref}}" - brname="${brname##refs/heads/}" - brdir=${brname//\//--} # replace '/' with '--' - rm -rf _gh-pages/branch/${brdir} - rsync -a _build/dirhtml/ _gh-pages/branch/${brdir} - # Go through each branch in _gh-pages/branch/, if it's not a - # ref, then delete it. - - name: Delete old feature branches - if: ${{ github.event_name == 'push' }} - run: | - set -x - for brdir in `ls _gh-pages/branch/` ; do - brname=${brdir//--/\/} # replace '--' with '/' - if ! git show-ref remotes/origin/$brname ; then - echo "Removing $brdir" - rm -r _gh-pages/branch/$brdir/ - fi - done + + # Use gh-pages-multibranch to multiplex different branches into + # one deployment. See + # https://github.com/coderefinery/gh-pages-multibranch + - name: gh-pages multibranch + uses: coderefinery/gh-pages-multibranch@main + if: ${{ github.event_name == 'push' && env.MULTIBRANCH == 'true' }} + with: + directory: _gh-pages/ + default_branch: ${{ env.DEFAULT_BRANCH }} + publish_branch: gh-pages # Add the .nojekyll file - name: nojekyll @@ -134,15 +128,43 @@ jobs: run: | touch _gh-pages/.nojekyll + # Save artifact for the next step. + - uses: actions/upload-artifact@v4 + if: ${{ github.event_name == 'push' }} + with: + name: gh-pages-build + path: _gh-pages/ + + # Deploy in a separate job so that write permissions are restricted + # to the minimum steps. + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: build + # This if can't use the env context - find better way later. + if: ${{ github.event_name == 'push' }} + permissions: + contents: write + + steps: + - uses: actions/download-artifact@v4 + if: ${{ github.event_name == 'push' && ( env.MULTIBRANCH == 'true' || github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH )) }} + with: + name: gh-pages-build + path: _gh-pages/ + + # As of 2023, we could publish to pages via a Deployment. This + # isn't done yet to give it time to stabilize (out of beta), and + # also having a gh-pages branch to check out is rather + # convenient. + # Deploy # https://github.com/peaceiris/actions-gh-pages - name: Deploy uses: peaceiris/actions-gh-pages@v3 - if: ${{ github.event_name == 'push' }} - #if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/${{ env.DEFAULT_BRANCH }}' }} + if: ${{ github.event_name == 'push' && ( env.MULTIBRANCH == 'true' || github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH )) }} with: publish_branch: gh-pages github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: _gh-pages/ force_orphan: true -