diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 8d16ada..24247ce 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -1,54 +1,69 @@ -# From: https://github.com/rkdarst/sphinx-actions-test/blob/master/.github/workflows/sphinx-build.yml +# Deploy Sphinx. This could be shorter, but we also do some extra +# stuff. +# +# License: CC-0. This is the canonical location of this file, which +# you may want to link to anyway: +# https://github.com/coderefinery/sphinx-lesson-template/blob/main/.github/workflows/sphinx.yml +# https://raw.githubusercontent.com/coderefinery/sphinx-lesson-template/main/.github/workflows/sphinx.yml + name: sphinx on: [push, pull_request] -# If these SPHINXOPTS are enabled, then be strict about the builds and -# fail on any warnings -#env: -# SPHINXOPTS: "-W --keep-going -T" env: - DEFAULT_BRANCH: main + DEFAULT_BRANCH: "main" + # If these SPHINXOPTS are enabled, then be strict about the + # builds and fail on any warnings. + #SPHINXOPTS: "-W --keep-going -T" + GENERATE_PDF: true # to enable, must be 'true' lowercase + GENERATE_SINGLEHTML: true # to enable, must be 'true' lowercase + PDF_FILENAME: lesson.pdf + MULTIBRANCH: true # to enable, must be 'true' lowercase jobs: - build-and-deploy: - name: Build and gh-pages + build: + name: Build runs-on: ubuntu-latest + permissions: + contents: read + steps: # https://github.com/marketplace/actions/checkout - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 lfs: true + # https://github.com/marketplace/actions/setup-python # ^-- This gives info on matrix testing. - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - # https://docs.github.com/en/actions/guides/building-and-testing-python#caching-dependencies - # ^-- How to set up caching for pip on Ubuntu - - name: Cache pip - uses: actions/cache@v2 + uses: actions/setup-python@v4 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + python-version: '3.11' + cache: 'pip' + # https://docs.github.com/en/actions/guides/building-and-testing-python#installing-dependencies # ^-- This gives info on installing dependencies with pip - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt + + # Debug - name: Debugging information + env: + ref: ${{github.ref}} + event_name: ${{github.event_name}} + head_ref: ${{github.head_ref}} + base_ref: ${{github.base_ref}} run: | - echo "github.ref:" ${{github.ref}} - echo "github.event_name:" ${{github.event_name}} - echo "github.head_ref:" ${{github.head_ref}} - echo "github.base_ref:" ${{github.base_ref}} + echo "github.ref: ${ref}" + echo "github.event_name: ${event_name}" + echo "github.head_ref: ${head_ref}" + echo "github.base_ref: ${base_ref}" + echo "GENERATE_PDF: ${GENERATE_PDF}" + echo "GENERATE_SINGLEHTML: ${GENERATE_SINGLEHTML}" set -x git rev-parse --abbrev-ref HEAD git branch @@ -61,7 +76,8 @@ jobs: # Build - uses: ammaraskar/sphinx-problem-matcher@master - - name: Build Sphinx docs + - name: Build Sphinx docs (dirhtml) + # SPHINXOPTS used via environment variables run: | make dirhtml # This fixes broken copy button icons, as explained in @@ -73,60 +89,38 @@ jobs: # https://github.com/readthedocs/sphinx_rtd_theme/pull/1025 sed -i 's/url_root="#"/url_root=""/' _build/dirhtml/index.html || true + # singlehtml + - name: Generate singlehtml + if: ${{ env.GENERATE_SINGLEHTML == 'true' }} + run: | + make singlehtml + mv _build/singlehtml/ _build/dirhtml/singlehtml/ - # The following supports building all branches and combining on - # gh-pages + # PDF if requested + - name: Generate PDF + if: ${{ env.GENERATE_PDF == 'true' }} + run: | + pip install https://github.com/rkdarst/sphinx_pyppeteer_builder/archive/refs/heads/main.zip + make pyppeteer + mv _build/pyppeteer/*.pdf _build/dirhtml/${PDF_FILENAME} - # Clone and set up the old gh-pages branch - - name: Clone old gh-pages + # Stage all deployed assets in _gh-pages/ for simplicity, and to + # prepare to do a multi-branch deployment. + - name: Copy deployment data to _gh-pages/ if: ${{ github.event_name == 'push' }} - run: | - set -x - git fetch - ( git branch gh-pages remotes/origin/gh-pages && git clone . --branch=gh-pages _gh-pages/ ) || mkdir _gh-pages - rm -rf _gh-pages/.git/ - mkdir -p _gh-pages/branch/ - # If a push and default branch, copy build to _gh-pages/ as the "main" - # deployment. - - name: Copy new build (default branch) - if: | - contains(github.event_name, 'push') && - contains(github.ref, env.DEFAULT_BRANCH) - run: | - set -x - # Delete everything under _gh-pages/ that is from the - # primary branch deployment. Eicludes the other branches - # _gh-pages/branch-* paths, and not including - # _gh-pages itself. - find _gh-pages/ -mindepth 1 ! -path '_gh-pages/branch*' -delete + run: rsync -a _build/dirhtml/ _gh-pages/ - # If a push and not on default branch, then copy the build to - # _gh-pages/branch/$brname (transforming '/' into '--') - - name: Copy new build (branch) - if: | - contains(github.event_name, 'push') && - !contains(github.ref, env.DEFAULT_BRANCH) - run: | - set -x - #brname=$(git rev-parse --abbrev-ref HEAD) - brname="${{github.ref}}" - brname="${brname##refs/heads/}" - brdir=${brname//\//--} # replace '/' with '--' - rm -rf _gh-pages/branch/${brdir} - rsync -a _build/dirhtml/ _gh-pages/branch/${brdir} - # Go through each branch in _gh-pages/branch/, if it's not a - # ref, then delete it. - - name: Delete old feature branches - if: ${{ github.event_name == 'push' }} - run: | - set -x - for brdir in `ls _gh-pages/branch/` ; do - brname=${brdir//--/\/} # replace '--' with '/' - if ! git show-ref remotes/origin/$brname ; then - echo "Removing $brdir" - rm -r _gh-pages/branch/$brdir/ - fi - done + + # Use gh-pages-multibranch to multiplex different branches into + # one deployment. See + # https://github.com/coderefinery/gh-pages-multibranch + - name: gh-pages multibranch + uses: coderefinery/gh-pages-multibranch@main + if: ${{ github.event_name == 'push' && env.MULTIBRANCH == 'true' }} + with: + directory: _gh-pages/ + default_branch: ${{ env.DEFAULT_BRANCH }} + publish_branch: gh-pages # Add the .nojekyll file - name: nojekyll @@ -134,15 +128,43 @@ jobs: run: | touch _gh-pages/.nojekyll + # Save artifact for the next step. + - uses: actions/upload-artifact@v4 + if: ${{ github.event_name == 'push' }} + with: + name: gh-pages-build + path: _gh-pages/ + + # Deploy in a separate job so that write permissions are restricted + # to the minimum steps. + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: build + # This if can't use the env context - find better way later. + if: ${{ github.event_name == 'push' }} + permissions: + contents: write + + steps: + - uses: actions/download-artifact@v4 + if: ${{ github.event_name == 'push' && ( env.MULTIBRANCH == 'true' || github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH )) }} + with: + name: gh-pages-build + path: _gh-pages/ + + # As of 2023, we could publish to pages via a Deployment. This + # isn't done yet to give it time to stabilize (out of beta), and + # also having a gh-pages branch to check out is rather + # convenient. + # Deploy # https://github.com/peaceiris/actions-gh-pages - name: Deploy uses: peaceiris/actions-gh-pages@v3 - if: ${{ github.event_name == 'push' }} - #if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/${{ env.DEFAULT_BRANCH }}' }} + if: ${{ github.event_name == 'push' && ( env.MULTIBRANCH == 'true' || github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH )) }} with: publish_branch: gh-pages github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: _gh-pages/ force_orphan: true - diff --git a/.gitignore b/.gitignore index c0d3369..078e14b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ /venv* .jupyter_cache jupyter_execute + +# pixi environments +.pixi +*.egg-info diff --git a/LICENSE.code b/LICENSE.code index 3f02770..6393229 100644 --- a/LICENSE.code +++ b/LICENSE.code @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022, HPDA-Python and individual contributors. +Copyright (c) 2024, ENCCS and individual contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/Makefile b/Makefile index d50f54e..46f35b9 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,7 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Live reload site documents for local development +livehtml: + sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/content/_static/overrides.css b/content/_static/overrides.css index 26a0f16..56413f4 100644 --- a/content/_static/overrides.css +++ b/content/_static/overrides.css @@ -26,7 +26,7 @@ background: #DDDDDD; } .rst-content .instructor-note > .admonition-title { - background: #BBBBBB; + background: #595959; } .rst-content .instructor-note > .admonition-title::before { content: ""; @@ -37,7 +37,7 @@ background: #EEEEBB; } .rst-content .callout > .admonition-title { - background: #BBCC33; + background: #595959; } /* questions */ @@ -45,15 +45,15 @@ background: rgba(253, 219, 199, 0.3); } .rst-content .questions > .admonition-title { - background: rgba(204, 51, 17, 0.5); + background: #993360; } /* discussion */ .rst-content .discussion { - background: rgba(231, 212, 232 0.3); + background: rgba(231, 212, 232, 0.3); } .rst-content .discussion > .admonition-title { - background: rgba(194, 165, 207, 0.5); + background: #714884; } /* signature */ @@ -61,7 +61,7 @@ background: rgba(217, 240, 211, 0.3); } .rst-content .signature > .admonition-title { - background: rgba(172, 211, 158, 0.5); + background: #21634b; } .rst-content .signature > .admonition-title::before { content: "\01F527"; @@ -72,7 +72,7 @@ background: rgba(217, 240, 211, 0.0); } .rst-content .parameters > .admonition-title { - background: rgba(172, 211, 158, 0.5); + background: #21634b; } .rst-content .parameters > .admonition-title::before { content: "\01F4BB"; @@ -83,7 +83,7 @@ background: rgba(221, 221, 221, 0.3); } .rst-content .typealong > .admonition-title { - background: rgba(187, 187, 187, 1.0); + background: rgba(89, 89, 89, 1.0); } .rst-content .typealong > .admonition-title::before { content: "\02328"; diff --git a/content/conf.py b/content/conf.py index 54867ca..b3452fd 100644 --- a/content/conf.py +++ b/content/conf.py @@ -21,7 +21,7 @@ copyright = "2022, ENCCS and individual contributors." author = "ENCCS and individual contributors." github_user = "ENCCS" -github_repo_name = "HPDA-Python" +github_repo_name = "" # auto-detected from dirname if blank github_version = "main" conf_py_path = "/content/" # with leading and trailing slash @@ -44,7 +44,15 @@ # jupyter_execute_notebooks = "off" # jupyter_execute_notebooks = "auto" # *only* execute if at least one output is missing. # jupyter_execute_notebooks = "force" -jupyter_execute_notebooks = "cache" +nb_execution_mode = "cache" + +# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html +myst_enable_extensions = [ + "colon_fence", +] + +# Settings for sphinx-copybutton +copybutton_exclude = ".linenos, .gp" # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] @@ -73,7 +81,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = ["overrides.css"] # HTML context: @@ -84,7 +92,7 @@ "github_user": github_user, # Auto-detect directory name. This can break, but # useful as a default. - "github_repo": github_repo_name or basename(dirname(realpath(__file__))), + "github_repo": github_repo_name or basename(dirname(dirname(realpath(__file__)))), "github_version": github_version, "conf_py_path": conf_py_path, } @@ -120,17 +128,27 @@ class TypealongDirective(_BaseCRDirective): extra_classes = ["toggle-shown", "dropdown"] -DIRECTIVES = [SignatureDirective, ParametersDirective, TypealongDirective] +DIRECTIVES: list[type[_BaseCRDirective]] = [ + SignatureDirective, + ParametersDirective, + TypealongDirective, +] def setup(app): for obj in DIRECTIVES: - app.add_directive(obj.get_cssname(), obj) + app.add_directive(obj.cssname(), obj) + import os -if os.environ.get('GITHUB_REF', '') == 'refs/heads/main': + +if os.environ.get("GITHUB_REF", "") == "refs/heads/main": html_js_files = [ - ('https://plausible.io/js/script.js', {"data-domain": "enccs.github.io/hpda-python", "defer": "\ -defer"}), + ( + "https://plausible.io/js/script.js", + { + "data-domain": "enccs.github.io/hpda-python", + "defer": "defer", + }, + ), ] - diff --git a/content/env/environment.yml b/content/env/environment.yml index 703afb0..33d4c95 100644 --- a/content/env/environment.yml +++ b/content/env/environment.yml @@ -1,8 +1,6 @@ name: pyhpda channels: - conda-forge - - defaults - - bioconda dependencies: - numpy - scipy @@ -15,13 +13,14 @@ dependencies: - netcdf4 - matplotlib - dask + - mpi4py + - scalene - graphviz - python-graphviz - ipywidgets - widgetsnbextension - dask-jobqueue - dask-labextension - - snakemake-minimal - xarray - jupyter - jupyterlab diff --git a/content/index.rst b/content/index.rst index a6322ee..f591db4 100644 --- a/content/index.rst +++ b/content/index.rst @@ -62,15 +62,15 @@ and distributed computing. performance-boosting dask - - .. toctree:: :maxdepth: 1 :caption: Optional material + setup-eurohpc pandas-extra GPU-computing + parallel-computing_opt .. toctree:: diff --git a/content/parallel-computing_opt.rst b/content/parallel-computing_opt.rst index fe2beb6..a73de7d 100644 --- a/content/parallel-computing_opt.rst +++ b/content/parallel-computing_opt.rst @@ -1,5 +1,10 @@ - # on some HPC systems you might need 'srun -n 4' instead of 'mpirun -np 4' - # on Vega, add this module for MPI libraries: ml add foss/2020b +More on parallel computing +========================== + +.. note:: + + on some HPC systems you might need ``srun -n 4`` instead of ``mpirun -np 4`` + on Vega, add this module for MPI libraries: ``ml add foss/2020b`` .. callout:: MPI libraries diff --git a/content/setup-eurohpc.rst b/content/setup-eurohpc.rst new file mode 100644 index 0000000..4182d33 --- /dev/null +++ b/content/setup-eurohpc.rst @@ -0,0 +1,246 @@ +Installation in EuroHPC systems +------------------------------- + +.. warning:: + + These instructions may be outdated and were last updated in 2023. + + +Here are instructions for accessing the EuroHPC system, setting up the Python environment +and running jobs. Please follow the instructions for the HPC system that will be used +during the workshop that you are attending. + +.. tabs:: + + .. group-tab:: Vega + + Thanks to `IZUM `__ in Slovenia we will have an allocation on the + petascale `Vega `__ EuroHPC system for the duration of the workshop. + The sustained and peak performance of Vega is 6.9 petaflops and 10.1 petaflops, respectively. + + **Architecture**: + + Vega has both `GPU and CPU partititions `__: + + - CPU partition: Each node has two AMD Epyc 7H12 CPUs, each with 64 cores. 768 nodes with 256 GB, + 192 nodes with 1 TB of RAM DDR4-3200, local 1.92 TB M.2 SSD. + - GPU partition: Each node has 4 GPUs NVidia A100 with 40 GB HBMI2 and two AMD Epyc 7H12 CPUs. + In total 60 nodes with 512 GB of RAM DDR4-3200, local 1.92 TB M.2 SSD + + .. group-tab:: Karolina + + Thanks to `IT4I `__ in the Czech Republic we will have an allocation + on the petascale `Karolina supercomputer `__ + for the duration of the workshop. The peak performance of Karolina is 15.7 petaflops. + + **Architecture**: + + - 720x 2x AMD 7H12, 64 cores, 2,6 GHz, 92,160 cores in total + - 72x 2x AMD 7763, 64 cores, 2,45 GHz, 9,216 cores in total + - 72x 8x NVIDIA A100 GPU, 576 GPU in total + - 32x Intel Xeon-SC 8628, 24 cores, 2,9 GHz, 768 cores in total + - 36x 2x AMD 7H12, 64 cores, 2,6 GHz, 4,608 cores in total + - 2x 2x AMD 7452, 32 cores, 2,35 GHz, 128 cores in total + +Software on the cluster is available through a module system. +First load the Anaconda module to get access to the ``conda`` package manager: + +.. tabs:: + + .. group-tab:: Vega + + .. code-block:: console + + $ #check available Anaconda modules: + $ ml av Anaconda3 + $ ml add Anaconda3/2020.11 + + .. group-tab:: Karolina + + .. code-block:: console + + $ #check available Anaconda modules: + $ ml av Anaconda3 + $ ml add Anaconda3/2021.11 + + +To be able to create conda environments in your home directory you need to initialize it. +The following command adds the necessary configuration to your ``.bashrc`` file: + +.. code-block:: console + + $ conda init bash + +You now need to either log in to the cluster again or start a new shell session by typing ``bash``: + +.. code-block:: console + + $ bash + +Now, either create a new environment with all required dependencies or activate +a pre-existing environment created in a directory you have access to: + +.. tabs:: + + .. tab:: Create new environment in $HOME + + .. code-block:: console + + $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml + + The installation can take several minutes. + Now activate the environment by: + + .. code-block:: console + + $ conda activate pyhpda + + .. tab:: Activate existing environment + + .. code-block:: console + + $ conda activate /path/to/envdir/ + + +mpi4py +^^^^^^ + +Additional steps are required to use mpi4py since the Python package needs to be +linked with the system's MPI libraries. + +.. tabs:: + + .. group-tab:: Vega + + To use mpi4py you need to load a module which contains MPI libraries and then install ``mpi4py`` + using ``pip``: + + .. code-block:: console + + $ ml add foss/2020b + $ CC=gcc MPICC=mpicc python3 -m pip install mpi4py --no-binary=mpi4py + + .. group-tab:: Karolina + + To use mpi4py you only need to load a module: + + .. code-block:: console + + $ ml add mpi4py/3.1.1-gompi-2020b + +Running jobs +^^^^^^^^^^^^ + +Resources can be allocated both through batch jobs (submitting a script to the scheduler) +and interactively. You will need to provide a project ID when asking for an allocation. +To find out what projects you belong to on the cluster, type: + +.. code-block:: console + + $ sacctmgr -p show associations user=$USER + +The second column of the output contains the project ID. + +.. tabs:: + + .. group-tab:: Vega + + Vega uses the SLURM scheduler. + Use the following command to allocate one interactive node with 8 cores for 1 hour + in the CPU partition. If there is a reservation on the cluster for the workshop, + add ``--reservation=RESERVATIONNAME`` to the command. + + .. code-block:: console + + $ salloc -N 1 --ntasks-per-node=8 --ntasks-per-core=1 -A --partition=cpu -t 01:00:00 + + To instead book a GPU node, type (again adding reservation flag if relevant): + + .. code-block:: console + + $ salloc -N 1 --ntasks-per-node=1 --ntasks-per-core=1 -A --partition=gpu --gres=gpu:1 --cpus-per-task 1 -t 01:00:00 + + .. group-tab:: Karolina + + Karolina uses the PBS scheduler. + To allocate one interactive node for + 1 hour on 1 node in the CPU partition and express queue: + + .. code-block:: console + + $ qsub -A DD-22-28 -q qexp -l walltime=01:00:00 -I + +Running Jupyter +^^^^^^^^^^^^^^^ + +The following procedure starts a Jupyter-Lab server on a compute node, creates an SSH tunnel from +your local machine to the compute node, and then connects to the remote Jupyter-Lab server from your +browser. + +First make sure to follow the above instructions to: + +- Allocate an interactive compute node for a sufficiently long time +- Switch to the pyhpda conda environment. + +After allocating an interactive node you will see the name of the node in the output. + +.. tabs:: + + .. group-tab:: Vega + + After allocating an interactive node you will see the name of the node in the + output, e.g. ``salloc: Nodes cn0709 are ready for job``. + + You now need to ssh to that node, switch to the pyhpda + conda environment, and start the Jupyter-Lab server on a particular port + (choose one between 8000 and 9000) + and IP address (the name of the compute node). Also load a module containing + OpenMPI to have access to MPI inside Jupyter: + + .. code-block:: console + + $ ssh cn0709 + $ conda activate pyhpda + $ ml add foss/2021b + $ jupyter-lab --no-browser --port=8123 --ip=cn0709 + + .. group-tab:: Karolina + + After allocating an interactive node your terminal session will be connected to that node. + Find out the name of your compute node. Your terminal prompt should show it but you can + also run the hostname command. Look only at the node name (e.g. cn012) and disregard + the ``.karolina.it4i.cz`` part. + + Now start the Jupyter-Lab server on a particular port + (choose one between 8000 and 9000) + and IP address (the name of the compute node): + + .. code-block:: console + + $ jupyter-lab --no-browser --port=8123 --ip=cn012 + + +Now create an SSH tunnel **from a new terminal on your local machine** to the correct +port and IP: + +.. tabs:: + + .. group-tab:: Vega + + .. code-block:: console + + $ ssh -TN -f YourUsername@login.vega.izum.si -L localhost:8123:cn0709:8123 -L localhost:8787:cn0709:8787 + + .. group-tab:: Karolina + + .. code-block:: console + + $ ssh -TN -f YourUsername@login2.karolina.it4i.cz -L localhost:8123:cn012:8123 + +Go back to the terminal running Jupyter-Lab on the compute node, and copy-paste the URL +starting with ``127.0.0.1`` which contains a long token into your local browser. +If that does not work, try replacing ``127.0.0.1`` with ``localhost``. + +If everything is working as it should, you should now be able to create a new Jupyter notebook in your browser +which is connected to the compute node and the ``pyhpda`` conda environment. + diff --git a/content/setup.rst b/content/setup.rst index 35f079c..6051e3e 100644 --- a/content/setup.rst +++ b/content/setup.rst @@ -2,35 +2,33 @@ Installation and HPC access =========================== This page contains instructions for installing the required dependencies on a local computer -as well as instructions for logging in and using two EuroHPC systems. +as well as instructions for logging into a EuroHPC system. Local installation ------------------ -If you already have a preferred way to manage Python versions and -libraries, you can stick to that. If not, we recommend that you -install Python3 and all libraries using -`miniconda `__, -a free minimal installer for the package, dependency and environment manager +If you already have a preferred way to manage Python versions and +libraries, you can stick to that. If not, we recommend that you install +Python3 and all libraries using +`Miniforge `__, a free +minimal installer for the package, dependency and environment manager `conda `__. -Please follow the installation instructions on -https://docs.conda.io/en/latest/miniconda.html to install Miniconda3. +Please follow the installation instructions on +https://conda-forge.org/download/ to install Miniforge. -Make sure that both Python and conda are correctly installed: +Make sure that conda is correctly installed: .. code-block:: console - $ python --version - $ # should give something like Python 3.9.7 $ conda --version - $ # should give something like conda 4.10.2 + conda 24.11.2 With conda installed, install the required dependencies by running: .. code-block:: console - $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml + $ conda env create --yes -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml This will create a new environment ``pyhpda`` which you need to activate by: @@ -38,12 +36,19 @@ This will create a new environment ``pyhpda`` which you need to activate by: $ conda activate pyhpda -To use MPI4Py on your computer you need to install MPI libraries. With conda, these libraries are -installed automatically when installing the mpi4py package: +.. To use MPI4Py on your computer you need to install MPI libraries. With conda, these libraries are +.. installed automatically when installing the mpi4py package: +.. +.. .. code-block:: console +.. +.. $ conda install -c conda-forge mpi4py + +Ensure that the Python version is fairly recent: .. code-block:: console - $ conda install -c conda-forge mpi4py + $ python --version + Python 3.12.8 Finally, open Jupyter-Lab in your browser: @@ -52,244 +57,12 @@ Finally, open Jupyter-Lab in your browser: $ jupyter-lab -EuroHPC systems ---------------- - -Here are instructions for accessing the EuroHPC system, setting up the Python environment -and running jobs. Please follow the instructions for the HPC system that will be used -during the workshop that you are attending. - -.. tabs:: - - .. group-tab:: Vega - - Thanks to `IZUM `__ in Slovenia we will have an allocation on the - petascale `Vega `__ EuroHPC system for the duration of the workshop. - The sustained and peak performance of Vega is 6.9 petaflops and 10.1 petaflops, respectively. - - **Architecture**: - - Vega has both `GPU and CPU partititions `__: - - - CPU partition: Each node has two AMD Epyc 7H12 CPUs, each with 64 cores. 768 nodes with 256 GB, - 192 nodes with 1 TB of RAM DDR4-3200, local 1.92 TB M.2 SSD. - - GPU partition: Each node has 4 GPUs NVidia A100 with 40 GB HBMI2 and two AMD Epyc 7H12 CPUs. - In total 60 nodes with 512 GB of RAM DDR4-3200, local 1.92 TB M.2 SSD - - .. group-tab:: Karolina - - Thanks to `IT4I `__ in the Czech Republic we will have an allocation - on the petascale `Karolina supercomputer `__ - for the duration of the workshop. The peak performance of Karolina is 15.7 petaflops. - - **Architecture**: - - - 720x 2x AMD 7H12, 64 cores, 2,6 GHz, 92,160 cores in total - - 72x 2x AMD 7763, 64 cores, 2,45 GHz, 9,216 cores in total - - 72x 8x NVIDIA A100 GPU, 576 GPU in total - - 32x Intel Xeon-SC 8628, 24 cores, 2,9 GHz, 768 cores in total - - 36x 2x AMD 7H12, 64 cores, 2,6 GHz, 4,608 cores in total - - 2x 2x AMD 7452, 32 cores, 2,35 GHz, 128 cores in total - -Software on the cluster is available through a module system. -First load the Anaconda module to get access to the ``conda`` package manager: - -.. tabs:: - - .. group-tab:: Vega - - .. code-block:: console - - $ #check available Anaconda modules: - $ ml av Anaconda3 - $ ml add Anaconda3/2020.11 - - .. group-tab:: Karolina - - .. code-block:: console - - $ #check available Anaconda modules: - $ ml av Anaconda3 - $ ml add Anaconda3/2021.11 - - -To be able to create conda environments in your home directory you need to initialize it. -The following command adds the necessary configuration to your ``.bashrc`` file: - -.. code-block:: console - - $ conda init bash - -You now need to either log in to the cluster again or start a new shell session by typing ``bash``: - -.. code-block:: console - - $ bash - -Now, either create a new environment with all required dependencies or activate -a pre-existing environment created in a directory you have access to: - -.. tabs:: - - .. tab:: Create new environment in $HOME - - .. code-block:: console - - $ conda env create -f https://raw.githubusercontent.com/ENCCS/hpda-python/main/content/env/environment.yml - - The installation can take several minutes. - Now activate the environment by: - - .. code-block:: console - - $ conda activate pyhpda - - .. tab:: Activate existing environment - - .. code-block:: console - - $ conda activate /path/to/envdir/ - - -mpi4py -^^^^^^ - -Additional steps are required to use mpi4py since the Python package needs to be -linked with the system's MPI libraries. - -.. tabs:: - - .. group-tab:: Vega - - To use mpi4py you need to load a module which contains MPI libraries and then install ``mpi4py`` - using ``pip``: - - .. code-block:: console - - $ ml add foss/2020b - $ CC=gcc MPICC=mpicc python3 -m pip install mpi4py --no-binary=mpi4py - - .. group-tab:: Karolina - - To use mpi4py you only need to load a module: - - .. code-block:: console - - $ ml add mpi4py/3.1.1-gompi-2020b - -Running jobs -^^^^^^^^^^^^ - -Resources can be allocated both through batch jobs (submitting a script to the scheduler) -and interactively. You will need to provide a project ID when asking for an allocation. -To find out what projects you belong to on the cluster, type: - -.. code-block:: console - - $ sacctmgr -p show associations user=$USER - -The second column of the output contains the project ID. - -.. tabs:: - - .. group-tab:: Vega - - Vega uses the SLURM scheduler. - Use the following command to allocate one interactive node with 8 cores for 1 hour - in the CPU partition. If there is a reservation on the cluster for the workshop, - add ``--reservation=RESERVATIONNAME`` to the command. - - .. code-block:: console - - $ salloc -N 1 --ntasks-per-node=8 --ntasks-per-core=1 -A --partition=cpu -t 01:00:00 - - To instead book a GPU node, type (again adding reservation flag if relevant): - - .. code-block:: console - - $ salloc -N 1 --ntasks-per-node=1 --ntasks-per-core=1 -A --partition=gpu --gres=gpu:1 --cpus-per-task 1 -t 01:00:00 - - .. group-tab:: Karolina - - Karolina uses the PBS scheduler. - To allocate one interactive node for - 1 hour on 1 node in the CPU partition and express queue: - - .. code-block:: console - - $ qsub -A DD-22-28 -q qexp -l walltime=01:00:00 -I - -Running Jupyter -^^^^^^^^^^^^^^^ - -The following procedure starts a Jupyter-Lab server on a compute node, creates an SSH tunnel from -your local machine to the compute node, and then connects to the remote Jupyter-Lab server from your -browser. - -First make sure to follow the above instructions to: - -- Allocate an interactive compute node for a sufficiently long time -- Switch to the pyhpda conda environment. - -After allocating an interactive node you will see the name of the node in the output. - -.. tabs:: - - .. group-tab:: Vega - - After allocating an interactive node you will see the name of the node in the - output, e.g. ``salloc: Nodes cn0709 are ready for job``. - - You now need to ssh to that node, switch to the pyhpda - conda environment, and start the Jupyter-Lab server on a particular port - (choose one between 8000 and 9000) - and IP address (the name of the compute node). Also load a module containing - OpenMPI to have access to MPI inside Jupyter: - - .. code-block:: console - - $ ssh cn0709 - $ conda activate pyhpda - $ ml add foss/2021b - $ jupyter-lab --no-browser --port=8123 --ip=cn0709 - - .. group-tab:: Karolina - - After allocating an interactive node your terminal session will be connected to that node. - Find out the name of your compute node. Your terminal prompt should show it but you can - also run the hostname command. Look only at the node name (e.g. cn012) and disregard - the ``.karolina.it4i.cz`` part. - - Now start the Jupyter-Lab server on a particular port - (choose one between 8000 and 9000) - and IP address (the name of the compute node): - - .. code-block:: console - - $ jupyter-lab --no-browser --port=8123 --ip=cn012 - - -Now create an SSH tunnel **from a new terminal on your local machine** to the correct -port and IP: - -.. tabs:: - - .. group-tab:: Vega - - .. code-block:: console - - $ ssh -TN -f YourUsername@login.vega.izum.si -L localhost:8123:cn0709:8123 -L localhost:8787:cn0709:8787 - - .. group-tab:: Karolina - - .. code-block:: console - - $ ssh -TN -f YourUsername@login2.karolina.it4i.cz -L localhost:8123:cn012:8123 +LUMI +------------ -Go back to the terminal running Jupyter-Lab on the compute node, and copy-paste the URL -starting with ``127.0.0.1`` which contains a long token into your local browser. -If that does not work, try replacing ``127.0.0.1`` with ``localhost``. +.. note:: -If everything is working as it should, you should now be able to create a new Jupyter notebook in your browser -which is connected to the compute node and the ``pyhpda`` conda environment. + .. todo:: + Add Instructions for using this + Go to LUMI `open OnDemand portal `__ diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..e49ae8a --- /dev/null +++ b/requirements.in @@ -0,0 +1,6 @@ +Sphinx +sphinx_rtd_theme>=2.0 +sphinx_rtd_theme_ext_color_contrast +myst_nb +sphinx-lesson>=0.8.19 +pillow diff --git a/requirements.txt b/requirements.txt index 135f7aa..6e084f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,260 @@ -Sphinx -sphinx_rtd_theme -sphinx_rtd_theme_ext_color_contrast -myst_nb -sphinx-lesson +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt +alabaster==0.7.16 + # via sphinx +anyio==4.8.0 + # via + # starlette + # watchfiles +asttokens==3.0.0 + # via stack-data +attrs==24.3.0 + # via + # jsonschema + # jupyter-cache + # referencing +babel==2.16.0 + # via sphinx +certifi==2024.12.14 + # via requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via + # jupyter-cache + # uvicorn +colorama==0.4.6 + # via sphinx-autobuild +comm==0.2.2 + # via ipykernel +debugpy==1.8.11 + # via ipykernel +decorator==5.1.1 + # via ipython +docutils==0.21.2 + # via + # myst-parser + # sphinx + # sphinx-rtd-theme + # sphinx-tabs + # sphinx-togglebutton +executing==2.1.0 + # via stack-data +fastjsonschema==2.21.1 + # via nbformat +greenlet==3.1.1 + # via sqlalchemy +h11==0.14.0 + # via uvicorn +idna==3.10 + # via + # anyio + # requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==8.5.0 + # via + # jupyter-cache + # myst-nb +ipykernel==6.29.5 + # via myst-nb +ipython==8.31.0 + # via + # ipykernel + # myst-nb +jedi==0.19.2 + # via ipython +jinja2==3.1.5 + # via + # myst-parser + # sphinx +jsonschema==4.23.0 + # via nbformat +jsonschema-specifications==2024.10.1 + # via jsonschema +jupyter-cache==1.0.1 + # via myst-nb +jupyter-client==8.6.3 + # via + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbformat +markdown-it-py==3.0.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==3.0.2 + # via jinja2 +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdit-py-plugins==0.4.2 + # via myst-parser +mdurl==0.1.2 + # via markdown-it-py +myst-nb==1.1.2 + # via + # -r requirements.in + # sphinx-lesson +myst-parser==4.0.0 + # via myst-nb +nbclient==0.10.2 + # via + # jupyter-cache + # myst-nb +nbformat==5.10.4 + # via + # jupyter-cache + # myst-nb + # nbclient +nest-asyncio==1.6.0 + # via ipykernel +packaging==24.2 + # via + # ipykernel + # sphinx +parso==0.8.4 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==11.1.0 + # via -r requirements.in +platformdirs==4.3.6 + # via jupyter-core +prompt-toolkit==3.0.48 + # via ipython +psutil==6.1.1 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.3 + # via stack-data +pygments==2.19.1 + # via + # ipython + # sphinx + # sphinx-tabs +python-dateutil==2.9.0.post0 + # via jupyter-client +pyyaml==6.0.2 + # via + # jupyter-cache + # myst-nb + # myst-parser +pyzmq==26.2.0 + # via + # ipykernel + # jupyter-client +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via sphinx +rpds-py==0.22.3 + # via + # jsonschema + # referencing +setuptools==75.7.0 + # via sphinx-togglebutton +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.4.7 + # via + # -r requirements.in + # myst-nb + # myst-parser + # sphinx-autobuild + # sphinx-copybutton + # sphinx-lesson + # sphinx-rtd-theme + # sphinx-rtd-theme-ext-color-contrast + # sphinx-tabs + # sphinx-togglebutton + # sphinxcontrib-jquery +sphinx-autobuild==2024.10.3 + # via sphinx-lesson +sphinx-copybutton==0.5.2 + # via sphinx-lesson +sphinx-lesson==0.8.19 + # via -r requirements.in +sphinx-minipres==0.2.1 + # via sphinx-lesson +sphinx-rtd-theme==3.0.2 + # via + # -r requirements.in + # sphinx-lesson +sphinx-rtd-theme-ext-color-contrast==0.3.2 + # via + # -r requirements.in + # sphinx-lesson +sphinx-tabs==3.4.7 + # via sphinx-lesson +sphinx-togglebutton==0.3.2 + # via sphinx-lesson +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +sqlalchemy==2.0.36 + # via jupyter-cache +stack-data==0.6.3 + # via ipython +starlette==0.45.2 + # via sphinx-autobuild +tabulate==0.9.0 + # via jupyter-cache +tornado==6.4.2 + # via + # ipykernel + # jupyter-client +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbformat +typing-extensions==4.12.2 + # via + # anyio + # ipython + # myst-nb + # sqlalchemy +urllib3==2.3.0 + # via requests +uvicorn==0.34.0 + # via sphinx-autobuild +watchfiles==1.0.3 + # via sphinx-autobuild +wcwidth==0.2.13 + # via prompt-toolkit +websockets==14.1 + # via sphinx-autobuild +wheel==0.45.1 + # via sphinx-togglebutton +zipp==3.21.0 + # via importlib-metadata