diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index d8b853121..7d9957bfc 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -71,7 +71,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 9a8affb11..afe12a2be 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -73,7 +73,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index dce1c009b..a18297339 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -1,4 +1,4 @@ - ### spack-stack-1.6.0 / skylab-7.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) + ### spack-stack-1.6.0 / skylab-7.x.y containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) specs: [base-env@1.0.0, jedi-base-env@1.0.0, ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.1, ecbuild@3.7.2, eccodes@2.32.0, ecflow@5, eckit@1.24.5, ecmwf-atlas@0.35.1 +fckit +trans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, diff --git a/configs/sites/aws-pcluster/config.yaml b/configs/sites/aws-pcluster/config.yaml index 3c8ac4c79..cf54e6360 100644 --- a/configs/sites/aws-pcluster/config.yaml +++ b/configs/sites/aws-pcluster/config.yaml @@ -1,9 +1,8 @@ config: - build_jobs: 24 + build_jobs: 8 # Overrides for spack build and staging areas to speed up builds # by using a local directory instead of the EFS shared filesystem - build_stage: /tmp/spack-stack/cache/build_stage test_stage: /tmp/spack-stack/cache/test_stage source_cache: /tmp/spack-stack/cache/source_cache diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml index f06ac3933..ec4dfb8ab 100644 --- a/configs/sites/derecho/packages.yaml +++ b/configs/sites/derecho/packages.yaml @@ -36,32 +36,22 @@ packages: ### All other external packages listed alphabetically autoconf: externals: - - spec: autoconf@2.71 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: autoconf@2.69 prefix: /usr automake: externals: - - spec: automake@1.16.5 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: automake@1.15.1 prefix: /usr binutils: externals: - spec: binutils@2.37.20211103 prefix: /usr - cmake: - externals: - - spec: cmake@3.26.3 - prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/cmake/3.26.3/gcc/7.5.0/k34x coreutils: externals: - spec: coreutils@8.32 prefix: /usr curl: externals: - - spec: curl@8.1.2+nghttp2 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: curl@7.79.1+gssapi+ldap+nghttp2 prefix: /usr cvs: @@ -83,10 +73,6 @@ packages: externals: - spec: findutils@4.8.0 prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /glade/u/apps/derecho/23.09/opt/view gawk: externals: - spec: gawk@4.2.1 @@ -94,13 +80,13 @@ packages: git: externals: - spec: git@2.41.0+tcltk - prefix: /glade/u/apps/derecho/23.09/opt/view + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git/2.41.0/gcc/7.5.0/jgni - spec: git@2.35.3+tcltk prefix: /usr git-lfs: externals: - spec: git-lfs@3.3.0 - prefix: /glade/u/apps/derecho/23.09/opt/view + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git-lfs/3.3.0/gcc/7.5.0/u3kh gmake: externals: - spec: gmake@4.2.1 @@ -113,21 +99,10 @@ packages: externals: - spec: krb5@1.19.2 prefix: /usr/lib/mit - libtool: - externals: - - spec: libtool@2.4.7 - prefix: /glade/u/apps/derecho/23.09/opt/view - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.18 prefix: /usr - # Automatically detected, but don't use - missing "ninja" dependency - #meson: - # externals: - # - spec: meson@1.2.0 - # prefix: /glade/u/apps/derecho/23.09/opt/view mysql: buildable: False externals: @@ -147,6 +122,10 @@ packages: externals: - spec: pkg-config@0.29.2 prefix: /usr + sed: + externals: + - spec: sed@4.4 + prefix: /usr subversion: externals: - spec: subversion@1.14.1 @@ -157,8 +136,6 @@ packages: prefix: /usr texinfo: externals: - - spec: texinfo@7.0.3 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: texinfo@6.5 prefix: /usr wget: diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml index f5dfcd3d2..6539d8472 100644 --- a/configs/sites/discover/packages.yaml +++ b/configs/sites/discover/packages.yaml @@ -121,10 +121,6 @@ packages: externals: - spec: libfuse@2.9.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/configs/sites/gaea-c5/packages.yaml b/configs/sites/gaea-c5/packages.yaml index 306fce195..0c7032604 100644 --- a/configs/sites/gaea-c5/packages.yaml +++ b/configs/sites/gaea-c5/packages.yaml @@ -122,10 +122,6 @@ packages: prefix: /usr libtirpc: variants: ~gssapi - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr # This package is currently incomplete (no headers), but still works libxaw: externals: diff --git a/configs/sites/hera/packages.yaml b/configs/sites/hera/packages.yaml index 75ec8fd4a..95c8e4e42 100644 --- a/configs/sites/hera/packages.yaml +++ b/configs/sites/hera/packages.yaml @@ -124,10 +124,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/configs/sites/hercules/compilers.yaml b/configs/sites/hercules/compilers.yaml index a86be4858..36c12beac 100644 --- a/configs/sites/hercules/compilers.yaml +++ b/configs/sites/hercules/compilers.yaml @@ -11,7 +11,10 @@ compilers: target: x86_64 modules: - intel-oneapi-compilers/2023.1.0 - environment: {} + environment: + set: + # https://github.com/ufs-community/ufs-weather-model/issues/2015#issuecomment-1864438186 + I_MPI_EXTRA_FILESYSTEM: 'ON' extra_rpaths: [] - compiler: spec: gcc@12.2.0 diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index c71b30cf1..b3deda8f5 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -3,6 +3,8 @@ packages: compiler:: [intel@2021.9.0, gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0, mvapich2@2.3.7] + # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing + #mpi:: [openmpi@4.1.6] ### MPI, Python, MKL mpi: @@ -20,11 +22,17 @@ packages: modules: - gcc/12.2.0 - mvapich2/2.3.7 + # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing + #openmpi: + # externals: + # - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath + # schedulers=slurm + # prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack + # modules: + # - gcc/12.2.0 + # - openmpi/4.1.6-gcc-12.2.0-spack ### Modifications of common packages - # DH* TRY IF THIS IS STILL TRUE # Version 2.0.8 doesn't compile on Hercules - # DH* TRY IF THIS IS STILL TRUE wgrib2: - # DH* TRY IF THIS IS STILL TRUE version:: ['3.1.1'] ### All other external packages listed alphabetically autoconf: @@ -84,10 +92,6 @@ packages: externals: - spec: groff@1.22.4 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.19 @@ -109,6 +113,10 @@ packages: prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd modules: - qt/5.15.8 + sed: + externals: + - spec: sed@4.8 + prefix: /usr subversion: externals: - spec: subversion@1.14.1 diff --git a/configs/sites/jet/packages.yaml b/configs/sites/jet/packages.yaml index 5f16ef38b..5ae9b49a6 100644 --- a/configs/sites/jet/packages.yaml +++ b/configs/sites/jet/packages.yaml @@ -129,10 +129,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 4102aa0cd..d279a5213 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -11,6 +11,7 @@ compilers:: modules: - PrgEnv-intel/8.3.2 - intel-classic/2021.4.0 + - node.js/20.10.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' @@ -31,6 +32,7 @@ compilers:: modules: - PrgEnv-gnu/8.3.2 - gcc/10.3.0 + - node.js/20.10.0 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib' diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 7188d7b4e..cdeebff1e 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -115,6 +115,10 @@ packages: prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 modules: - qt/5.15.2 + sed: + externals: + - spec: sed@4.4 + prefix: /usr subversion: externals: - spec: subversion@1.10.6 diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 233e6d993..78d2dfab1 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -103,10 +103,6 @@ packages: externals: - spec: groff@1.22.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.18 @@ -125,6 +121,10 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + sed: + externals: + - spec: sed@4.5 + prefix: /usr tar: externals: - spec: tar@1.30 diff --git a/configs/sites/noaa-aws/compilers.yaml b/configs/sites/noaa-aws/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-aws/compilers.yaml +++ b/configs/sites/noaa-aws/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-aws/packages.yaml b/configs/sites/noaa-aws/packages.yaml index 1cbe78222..3af5a9560 100644 --- a/configs/sites/noaa-aws/packages.yaml +++ b/configs/sites/noaa-aws/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 diff --git a/configs/sites/noaa-azure/compilers.yaml b/configs/sites/noaa-azure/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-azure/compilers.yaml +++ b/configs/sites/noaa-azure/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-azure/packages.yaml b/configs/sites/noaa-azure/packages.yaml index 886727adf..6cec5293b 100644 --- a/configs/sites/noaa-azure/packages.yaml +++ b/configs/sites/noaa-azure/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 @@ -50,10 +42,6 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - cmake: - externals: - - spec: cmake@3.27.2 - prefix: /contrib/spack-stack/cmake-3.27.2 cpio: externals: - spec: cpio@2.11 diff --git a/configs/sites/noaa-gcloud/compilers.yaml b/configs/sites/noaa-gcloud/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-gcloud/compilers.yaml +++ b/configs/sites/noaa-gcloud/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-gcloud/packages.yaml b/configs/sites/noaa-gcloud/packages.yaml index b3abc7a21..7a7215bd7 100644 --- a/configs/sites/noaa-gcloud/packages.yaml +++ b/configs/sites/noaa-gcloud/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 2a9ac00ec..e5b3f50be 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -30,6 +30,9 @@ packages: - openmpi/4.0.4 ### Modifications of common packages + # Temporary - see https://github.com/spack/spack/issues/41947 + cdo: + require: '@2.0.5' ### All other external packages listed alphabetically autoconf: @@ -125,10 +128,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/configs/templates/gsi-addon-dev/spack.yaml b/configs/templates/gsi-addon-dev/spack.yaml index 30464636c..0a8aedf6d 100644 --- a/configs/templates/gsi-addon-dev/spack.yaml +++ b/configs/templates/gsi-addon-dev/spack.yaml @@ -23,3 +23,9 @@ spack: - matrix: - [$packages] - [$compilers] + + packages: + python:: + require: ['@3.11.6'] + py-numpy:: + require: ['@1.23.4'] diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 54d6c1e69..8baebc82b 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -48,11 +48,3 @@ spack: exclude: # jedi-tools doesn't build with Intel - jedi-tools-env%intel - ## Don't even bother building those with Intel 18, - ## not needed. Many of the Python packages don't build. - #- ewok-env%intel@18 - #- jedi-fv3-env%intel@18 - #- jedi-tools-env%intel@18 - #- jedi-ufs-env%intel@18 - #- jedi-um-env%intel@18 - #- soca-env%intel@18 diff --git a/doc/modulefile_templates/node.js b/doc/modulefile_templates/node.js new file mode 100644 index 000000000..7f5fc113c --- /dev/null +++ b/doc/modulefile_templates/node.js @@ -0,0 +1,23 @@ +#%Module1.0 + +module-whatis "Provides an node.js-20.10.0 installation for use with spack." + +# Only allow one instance of compiler to load +conflict ecflow + +proc ModulesHelp { } { +puts stderr "Provides an node.js-20.10.0 installation for use with spack." +} + +if { [ module-info mode load ] && ![ is-loaded gcc/10.3.0 ] } { + module load gcc/10.3.0 +} + +# Set this value +set NODEJS_PATH "/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0" + +prepend-path PATH "${NODEJS_PATH}/bin" +prepend-path LD_LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path CPATH "${NODEJS_PATH}/include" +prepend-path CMAKE_PREFIX_PATH "${NODEJS_PATH}" diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 5235641c1..6cf8a46dc 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -270,7 +270,7 @@ mysql ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules``. mvapich2 - Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2``. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler. + Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2`` outside of spack and provide it as an external package. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler. .. code-block:: console @@ -290,6 +290,54 @@ mvapich2 make check make install +openmpi (currently only for testing) + Because of difficulties with the default ``openmpi`` on Hercules, we build ``openmpi`` outside of spack and provide it as an external package. It is necessary to load the ``gcc`` compiler module and the ``zlib`` module for consistency. The configuration options are mostly adopted from the default OpenMPI installations that were done by the system administrators using spack (many of them are default values), except that we use internal ``hwloc`` and ``pmix``. Create modulefile ``openmpi`` from template ``doc/modulefile_templates/openmpi``. + +.. code-block:: console + + ./configure \ + --enable-shared \ + --disable-silent-rules \ + --disable-builtin-atomics \ + --with-pmi=/opt/slurm \ + --enable-static \ + --enable-mpi1-compatibility \ + --without-hcoll \ + --without-psm2 \ + --without-knem \ + --without-verbs \ + --without-psm \ + --without-cma \ + --without-ucx \ + --without-mxm \ + --without-fca \ + --without-xpmem \ + --without-ofi \ + --without-cray-xpmem \ + --without-sge \ + --without-lsf \ + --without-loadleveler \ + --without-alps \ + --without-tm \ + --with-slurm \ + --disable-memchecker \ + --with-pmix=internal \ + --with-zlib=/apps/spack-managed/gcc-12.2.0/zlib-1.2.13-p3sxbyfgvvjy7jx4kizib2jwvhm4s6l4 \ + --with-hwloc=internal \ + --disable-java \ + --disable-mpi-java \ + --with-gpfs=no \ + --without-cuda \ + --enable-wrapper-rpath \ + --disable-wrapper-runpath \ + --disable-mpi-cxx \ + --disable-cxx-exceptions \ + --with-wrapper-ldflags="-Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib/gcc/x86_64-pc-linux-gnu/12.2.0 -Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib64" \ + --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack 2>&1 | tee log.config + make VERBOSE=1 -j4 + make check + make install + .. _MaintainersSection_Discover: ------------------------------ @@ -346,7 +394,7 @@ mysql NAVY HPCMP Narwhal ------------------------------ -On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. +On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. Also, temporarily it is necessary to install ``node.js`` as an external package to work around build errors for ``py-jupyter-server`` (see https://github.com/JCSDA/spack-stack/issues/928 and https://github.com/spack/spack/issues/41899). git-lfs The following instructions install ``git-lfs`` in ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0``. Version 2.10.0 is the default version for Narwhal. First, download the ``git-lfs`` RPM on a system with full internet access (e.g., Derecho) using ``wget https://download.opensuse.org/repositories/openSUSE:/Leap:/15.2/standard/x86_64/git-lfs-2.10.0-lp152.1.2.x86_64.rpm`` and copy this file to ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0/src``. Then switch to Narwhal and run the following commands. @@ -397,6 +445,26 @@ ecflow mysql ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31``. +node.js + ``node.js`` is difficult to install via ``spack``, but is needed to install certain Python packages. The complication is that when using a newer ``gcc`` compiler (either directly or as backend for ``icc`` etc.), the OS ``node.js`` errors out with unresolved symbols in the ``libstdc++`` library. Therefore, we need to install ``node.js`` with ``gcc@10.3.0`` loaded, and create modulefile ``node.js/20.10.0`` from template ``modulefiles/node.js``. + +.. code-block:: console + + module unload PrgEnv-cray + module load PrgEnv-gnu/8.3.2 + module unload gcc + module load gcc/10.3.0 + + mkdir -p node-js-20.10.0/src && cd node-js-20.10.0/src + wget https://nodejs.org/dist/v20.10.0/node-v20.10.0.tar.gz + tar -xvzf node-v20.10.0.tar.gz + cd node-v20.10.0/ + ./configure --partly-static \ + --prefix=/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0 \ + 2>&1 | tee log.config + make 2>&1 | tee log.make + make install 2>&1 | tee log.install + .. _MaintainersSection_Nautilus: ------------------------------ diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 2487bd4b8..783005b9a 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -13,13 +13,13 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Compiler | Versions tested/in use in one or more site configs | Spack compiler identifier | +===========================================+======================================================================+===========================+ -| Intel classic (icc, icpc, ifort) | 18.0.5.274 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +| Intel classic (icc, icpc, ifort) | 2021.3.0 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Intel mixed (icx, icpx, ifort) | all versions up to latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | GNU (gcc, g++, gfortran) | 9.2.0 to 12.2.0 (note: 13.x.y is **not** yet supported) | ``gcc@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ -| Apple clang (clang, clang++, w/ gfortran) | 10.0.0 to 14.0.3 | ``apple-clang@`` | +| Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 | ``apple-clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | LLVM clang (clang, clang++, w/ gfortran) | 10.0.0 to 14.0.3 | ``clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ @@ -453,6 +453,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack external find --scope system wget spack external find --scope system mysql spack external find --scope system texlive + spack external find --scope system sed 5. Find compilers, add to site config's ``compilers.yaml`` @@ -490,8 +491,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack config add "packages:fontconfig:variants:+pic" spack config add "packages:pixman:variants:+pic" spack config add "packages:cairo:variants:+pic" - spack config add "packages:libffi:version:['3.3']" - spack config add "packages:flex:version:['2.6.4']" 9. If you have manually installed lmod, you will need to update the site module configuration to use lmod instead of tcl. Skip this step if you followed the Ubuntu or Red Hat instructions above. diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 538a72ef8..947d39ffa 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,58 +8,58 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.5.1 installations (tier 1) +Officially supported spack-stack 1.6.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.5.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +Ready-to-use spack-stack 1.6.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS Applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers | +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules^* | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NASA | Discover^** | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env`` | Dom Heinzeller / ??? | +| MSU | Hercules gcc+openmpi alternative | GCC | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416`` | Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | ++---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ +| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Cam Book | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Hang Lei / Alex Richert | +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.1/envs/unified-env`` | Alex Richert / Dom Heinzeller | +| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Sarah King | +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Mark Potts | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | S4 | GCC | *currently not supported* | Dom Heinzeller / Mark Potts | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^* Uses a different ``wgrib2`` version 3.1.1 than the default 2.0.8. - -^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. +For more information about a specific platform, please see the individual sections below. For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. @@ -85,26 +85,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available +.. note:: + The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.6.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. + +.. note:: + spack-stack-1.6.0 on Orion provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + ------------------------------ MSU Hercules ------------------------------ @@ -117,27 +123,44 @@ The following is required for building new spack environments and for using spac module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load git-lfs/3.1.2 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-mvapich2/2.3.7 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available +For testing, an additional version of spack-stack-1.6.0 with GNU+OpenMPI is available. Load the following modules after loading mysql and ecflow: + +.. code-block:: console + + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416/install/modulefiles/Core + module load stack-gcc/12.2.0 + module load stack-openmpi/4.1.6 + module load stack-python/3.10.13 + module available + +.. note:: + spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + +.. note:: + spack-stack-1.6.0 on Hercules has additional packages `fms@2023.02.01`, `sp@2.3.0`, and `crtm@2.4.0` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. + .. _Preconfigured_Sites_Discover: ------------------------------ @@ -154,24 +177,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.3 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Narwhal: @@ -200,14 +223,14 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! @@ -229,14 +252,14 @@ With GNU, the following is required for building new spack environments and for module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.10.8 + module load stack-python/3.10.13 .. _Preconfigured_Sites_Nautilus: @@ -259,14 +282,14 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-openmpi/4.1.5rc2 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. @@ -286,7 +309,11 @@ With AMD clang/flang (aocc), the following is required for building new spack en .. note:: - ``spack-stack-1.5.1`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + ``spack-stack-1.6.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + +.. note:: + + `wgrib2@2.0.8` does not build on Nautilus, therefore we are using `wgrib2@3.1.1` on this system. .. note:: @@ -310,15 +337,15 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Derecho: @@ -339,24 +366,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.33 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading ecflow and mysql: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading ecflow and mysql: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading ecflow and mysql: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading ecflow and mysql: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. note:: @@ -368,7 +395,7 @@ For ``spack-stack-1.5.1`` with GNU, load the following modules after loading ecf NOAA Acorn (WCOSS2 test system) ------------------------------- -For spack-stack-1.5.1, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core``. +For spack-stack-1.6.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core``. On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! @@ -402,14 +429,14 @@ The following is required for building new spack environments and for using spac module load mysql/8.0.31 module load git-lfs/2.4.1 -For ``spack-stack-1.5.1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /contrib/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Gaea_C5: @@ -431,14 +458,14 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2023.1.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module -t available .. note:: @@ -467,28 +494,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.5.3 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. +.. note:: + + spack-stack-1.6.0 on Hera provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. _Preconfigured_Sites_Jet: ------------------------------ @@ -506,26 +537,30 @@ The following is required for building new spack environments and for using spac module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.8 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 module load stack-python/3.10.8 module available +.. note:: + + spack-stack-1.6.0 on Jet provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ @@ -540,20 +575,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 module unuse /opt/apps/modulefiles/Compiler/intel/22 module available Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. +.. note:: + + spack-stack-1.6.0 on S4 provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. note:: There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. @@ -564,49 +603,49 @@ Amazon Web Services Parallelcluster Ubuntu 20.04 Access to the JCSDA-managed AWS Parallel Clusters is not available to the public. The following instructions are for JCSDA core staff and in-kind contributors. -For ``spack-stack-1.5.1`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: +For ``spack-stack-1.6.0`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: .. code-block:: console module purge ulimit -s unlimited source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh - module use /mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.1.0 module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: +For ``spack-stack-1.6.0`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: module purge ulimit -s unlimited - module use /mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.4.0 module load stack-openmpi/4.1.4 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. note:: - The GNU stack is currently under testing and may not work as expected. We recommend using the well-tested Intel setup. + spack-stack-1.6.0 on AWS ParallelCluster has an additional package ``metis@5.1.0`` installed for both Intel and GNU. This is needed for MPAS, but must be loaded explictly (i.e. not part of ``jedi-mpas-env`` yet). ----------------------------- Amazon Web Services Red Hat 8 ----------------------------- -Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-6.1.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-7.1.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). -For ``spack-stack-1.5.1``, run: +For ``spack-stack-1.6.0``, run: .. code-block:: console ulimit -s unlimited scl_source enable gcc-toolset-11 - module use /home/ec2-user/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.2.1 module load stack-openmpi/4.1.5 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Configurable_Sites_CreateEnv: diff --git a/doc/source/conf.py b/doc/source/conf.py index 36b4daa50..ca4ccb5ff 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'spack-stack' -copyright = '2023 ' +copyright = '2024 ' author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' # The short X.Y version @@ -143,7 +143,7 @@ # Latex figure (float) alignment # # 'figure_align': 'htbp', - 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, 2023. spack-stack documentation release v1.5.1. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/v1.5.1/pdf/.}\sphinxmaketitle' + 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, 2024. spack-stack documentation release v1.6.0. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/v1.6.0/pdf/.}\sphinxmaketitle' } # Grouping the document tree into LaTeX files. List of tuples diff --git a/spack b/spack index cc9780cd0..2d27c7383 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit cc9780cd0e4f0a44137af65862d7a5d876d03fc8 +Subproject commit 2d27c7383b6ece7ce8ba5df57defdebd7180ea16