From f6df16501a2f43926aafe22ad1f9bb6f7118badb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Dec 2023 14:55:41 -0700 Subject: [PATCH 01/25] Mandatory updates for release/1.6.0 in .gitmodules, configs/containers/*.yaml, configs/containers/specs/jedi-ci.yaml, doc/source/conf.py --- .gitmodules | 2 +- configs/containers/docker-ubuntu-clang-mpich.yaml | 2 +- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- configs/containers/specs/jedi-ci.yaml | 2 +- doc/source/conf.py | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..17f5bc5fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/spack/spack #branch = develop url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + branch = release/1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index fd475d055..c31a1ac6d 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + ref: release/1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 7c2ca47b8..f7120b680 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -97,7 +97,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + ref: release/1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index abc283707..99f29349b 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + ref: release/1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index dce1c009b..a18297339 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -1,4 +1,4 @@ - ### spack-stack-1.6.0 / skylab-7.0.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) + ### spack-stack-1.6.0 / skylab-7.x.y containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) specs: [base-env@1.0.0, jedi-base-env@1.0.0, ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.1, ecbuild@3.7.2, eccodes@2.32.0, ecflow@5, eckit@1.24.5, ecmwf-atlas@0.35.1 +fckit +trans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, diff --git a/doc/source/conf.py b/doc/source/conf.py index 36b4daa50..0558e8818 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -20,14 +20,14 @@ # -- Project information ----------------------------------------------------- project = 'spack-stack' -copyright = '2023 ' +copyright = '2024 ' author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' # The short X.Y version -version = 'dev' +version = '1.6' # The full version, including alpha/beta/rc tags -release = 'develop' +release = '1.6.0' numfig = True @@ -143,7 +143,7 @@ # Latex figure (float) alignment # # 'figure_align': 'htbp', - 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, 2023. spack-stack documentation release v1.5.1. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/v1.5.1/pdf/.}\sphinxmaketitle' + 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, 2024. spack-stack documentation release v1.6.0. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/v1.6.0/pdf/.}\sphinxmaketitle' } # Grouping the document tree into LaTeX files. List of tuples From 859f66870a62da1c904c992e2c42d8e83409d9b6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Dec 2023 16:17:53 -0700 Subject: [PATCH 02/25] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17f5bc5fc..9f4516e45 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = release/1.6.0 + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = release/1.6.0 + url = https://github.com/climbfuji/spack + branch = feature/rel160_crtm_release [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 67633edb8..d71fff9b1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 67633edb8b36aa88ef488ba175ada8978192d562 +Subproject commit d71fff9b13cd0667e1c5a8d535e583dc0b31f1d1 From 0216729645aba10b8142fee525a2a0068744e9df Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Dec 2023 17:05:30 -0700 Subject: [PATCH 03/25] Generate module for qhull --- configs/common/modules_lmod.yaml | 1 - configs/common/modules_tcl.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index d8b853121..7d9957bfc 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -71,7 +71,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 9a8affb11..afe12a2be 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -73,7 +73,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline From 397410c0f7c696f9b56a1f9d4bb75d864e4748ef Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Dec 2023 10:54:09 -0700 Subject: [PATCH 04/25] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9f4516e45..17f5bc5fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = release/1.6.0 - url = https://github.com/climbfuji/spack - branch = feature/rel160_crtm_release + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = release/1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index d71fff9b1..4c6dcd2a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit d71fff9b13cd0667e1c5a8d535e583dc0b31f1d1 +Subproject commit 4c6dcd2a01927d06ccc1005d8403811d0eccdc30 From a1105bffdcf2e04433a1513cd80af8587c36d5ee Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Dec 2023 11:13:11 -0700 Subject: [PATCH 05/25] release/1.6.0: change crtm build type to release, remove qhull from module blacklist (#919) * Update submodule pointer for spack * Generate module for qhull --- configs/common/modules_lmod.yaml | 1 - configs/common/modules_tcl.yaml | 1 - spack | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index d8b853121..7d9957bfc 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -71,7 +71,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 9a8affb11..afe12a2be 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -73,7 +73,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/spack b/spack index 67633edb8..4c6dcd2a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 67633edb8b36aa88ef488ba175ada8978192d562 +Subproject commit 4c6dcd2a01927d06ccc1005d8403811d0eccdc30 From 4f0dcddcc7da1cc47c4d0c30b2b2c267585b4f26 Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Thu, 21 Dec 2023 17:15:26 -0500 Subject: [PATCH 06/25] set python 3.11.6 and corresponding py-numpy for gsi-addon-dev template (#920) Co-authored-by: Dom Heinzeller --- configs/templates/gsi-addon-dev/spack.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configs/templates/gsi-addon-dev/spack.yaml b/configs/templates/gsi-addon-dev/spack.yaml index 30464636c..0a8aedf6d 100644 --- a/configs/templates/gsi-addon-dev/spack.yaml +++ b/configs/templates/gsi-addon-dev/spack.yaml @@ -23,3 +23,9 @@ spack: - matrix: - [$packages] - [$compilers] + + packages: + python:: + require: ['@3.11.6'] + py-numpy:: + require: ['@1.23.4'] From b32f57190e29a5be1056047700ff335fd8e80c92 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 14:30:34 -0700 Subject: [PATCH 07/25] Update site config for Hercules for release/1.6.0 --- configs/sites/hercules/compilers.yaml | 5 ++++- configs/sites/hercules/packages.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/configs/sites/hercules/compilers.yaml b/configs/sites/hercules/compilers.yaml index a86be4858..36c12beac 100644 --- a/configs/sites/hercules/compilers.yaml +++ b/configs/sites/hercules/compilers.yaml @@ -11,7 +11,10 @@ compilers: target: x86_64 modules: - intel-oneapi-compilers/2023.1.0 - environment: {} + environment: + set: + # https://github.com/ufs-community/ufs-weather-model/issues/2015#issuecomment-1864438186 + I_MPI_EXTRA_FILESYSTEM: 'ON' extra_rpaths: [] - compiler: spec: gcc@12.2.0 diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index c71b30cf1..955bedfd7 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -84,10 +84,6 @@ packages: externals: - spec: groff@1.22.4 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.19 @@ -109,6 +105,10 @@ packages: prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd modules: - qt/5.15.8 + sed: + externals: + - spec: sed@4.8 + prefix: /usr subversion: externals: - spec: subversion@1.14.1 From 4599cf40da8706ca87bade9a651019118859d5a3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 15:58:52 -0700 Subject: [PATCH 08/25] Update documentation for spack-stack-1.6.0 for Hercules --- doc/source/PreConfiguredSites.rst | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 538a72ef8..e534a9b71 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,17 +8,17 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.5.1 installations (tier 1) +Officially supported spack-stack 1.6.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.5.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +Ready-to-use spack-stack 1.6.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS Applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers | +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules^* | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -61,6 +61,8 @@ Ready-to-use spack-stack 1.5.1 installations are available on the following, ful ^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. +For more information about a specific platform, please see the individual sections below. + For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. .. _Preconfigured_Sites_Tier1: @@ -118,26 +120,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-mvapich2/2.3.7 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available +.. note:: + spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`. + +.. note:: + spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two standard versions `fms@2023.04` and `fms@release-jcsda`. + .. _Preconfigured_Sites_Discover: ------------------------------ From bb14ba9d5b677742314424b5060905980db91c53 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Dec 2023 09:34:10 -0700 Subject: [PATCH 09/25] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17f5bc5fc..5127cacd5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = release/1.6.0 + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = release/1.6.0 + url = https://github.com/climbfuji/spack + branch = bugfix/py_gevent_intel [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 4c6dcd2a0..da5394638 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 4c6dcd2a01927d06ccc1005d8403811d0eccdc30 +Subproject commit da539463828f00e3d2fc5f2cddcd178e2d149bf0 From 096cd94e56b76cae8aca34b91c3cbe11d8e569b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Dec 2023 09:45:21 -0700 Subject: [PATCH 10/25] Update site config and documentation for Nautilus --- configs/sites/hercules/packages.yaml | 3 --- configs/sites/nautilus/packages.yaml | 8 ++++---- doc/source/PreConfiguredSites.rst | 21 ++++++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 955bedfd7..900b60913 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -22,9 +22,6 @@ packages: - mvapich2/2.3.7 ### Modifications of common packages - # DH* TRY IF THIS IS STILL TRUE # Version 2.0.8 doesn't compile on Hercules - # DH* TRY IF THIS IS STILL TRUE wgrib2: - # DH* TRY IF THIS IS STILL TRUE version:: ['3.1.1'] ### All other external packages listed alphabetically autoconf: diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 233e6d993..78d2dfab1 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -103,10 +103,6 @@ packages: externals: - spec: groff@1.22.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.18 @@ -125,6 +121,10 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + sed: + externals: + - spec: sed@4.5 + prefix: /usr tar: externals: - spec: tar@1.30 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e534a9b71..ff203126f 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -40,7 +40,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Sarah King | +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -57,8 +57,6 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^* Uses a different ``wgrib2`` version 3.1.1 than the default 2.0.8. - ^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. For more information about a specific platform, please see the individual sections below. @@ -119,6 +117,7 @@ The following is required for building new spack environments and for using spac module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load git-lfs/3.1.2 For ``spack-stack-1.6.0`` with Intel, load the following modules after loading mysql and ecflow: @@ -141,10 +140,10 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys module available .. note:: - spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`. + spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. note:: - spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two standard versions `fms@2023.04` and `fms@release-jcsda`. + spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. .. _Preconfigured_Sites_Discover: @@ -267,14 +266,14 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-openmpi/4.1.5rc2 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. @@ -294,7 +293,11 @@ With AMD clang/flang (aocc), the following is required for building new spack en .. note:: - ``spack-stack-1.5.1`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + ``spack-stack-1.6.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + +.. note:: + + `wgrib2@2.0.8` does not build on Nautilus, therefore we are using `wgrib2@3.1.1` on this system. .. note:: From f38e52e2b1e4693645b8069cb4a7e5236e5ae390 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 30 Dec 2023 19:07:05 -0700 Subject: [PATCH 11/25] Update site config for Discover and documentation for Discover and ParallelWorks --- configs/sites/discover/packages.yaml | 4 ---- doc/source/PreConfiguredSites.rst | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml index f5dfcd3d2..6539d8472 100644 --- a/configs/sites/discover/packages.yaml +++ b/configs/sites/discover/packages.yaml @@ -121,10 +121,6 @@ packages: externals: - spec: libfuse@2.9.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ff203126f..e1b1bd990 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -22,7 +22,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NASA | Discover^** | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env`` | Dom Heinzeller / ??? | +| NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -54,11 +54,9 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. - For more information about a specific platform, please see the individual sections below. For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. @@ -161,24 +159,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.3 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Narwhal: @@ -413,14 +411,14 @@ The following is required for building new spack environments and for using spac module load mysql/8.0.31 module load git-lfs/2.4.1 -For ``spack-stack-1.5.1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /contrib/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Gaea_C5: From b76ed225dd6ff133b1eac6020ea64a904c949fa1 Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:11:50 -0700 Subject: [PATCH 12/25] Update compiler compatibility in New Site documentation (#929) * Updated compatibility for apple clang compiler to 15.0.0 * Updated compiler compatibility for Intel Classic compilers according to versions used in the pre-configured sites. --- doc/source/NewSiteConfigs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 2487bd4b8..b8c39c421 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -13,13 +13,13 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Compiler | Versions tested/in use in one or more site configs | Spack compiler identifier | +===========================================+======================================================================+===========================+ -| Intel classic (icc, icpc, ifort) | 18.0.5.274 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +| Intel classic (icc, icpc, ifort) | 2021.3.0 to the latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | Intel mixed (icx, icpx, ifort) | all versions up to latest available version in oneAPI 2023.1.0 | ``intel@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | GNU (gcc, g++, gfortran) | 9.2.0 to 12.2.0 (note: 13.x.y is **not** yet supported) | ``gcc@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ -| Apple clang (clang, clang++, w/ gfortran) | 10.0.0 to 14.0.3 | ``apple-clang@`` | +| Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 | ``apple-clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ | LLVM clang (clang, clang++, w/ gfortran) | 10.0.0 to 14.0.3 | ``clang@`` | +-------------------------------------------+----------------------------------------------------------------------+---------------------------+ From 66dbe7dd5681ab3779db7dfdbd54a627e5d2fc52 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 13:12:49 -0700 Subject: [PATCH 13/25] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5127cacd5..17f5bc5fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = release/1.6.0 - url = https://github.com/climbfuji/spack - branch = bugfix/py_gevent_intel + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = release/1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index da5394638..46e4875a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit da539463828f00e3d2fc5f2cddcd178e2d149bf0 +Subproject commit 46e4875a0b7021094f0ddabae74e087c64849641 From 1eef392d2582ffb1df1369ca8bfcb32810db8910 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 13:13:40 -0700 Subject: [PATCH 14/25] Update site config for Narwhal --- configs/sites/narwhal/compilers.yaml | 2 ++ configs/sites/narwhal/packages.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 4102aa0cd..d279a5213 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -11,6 +11,7 @@ compilers:: modules: - PrgEnv-intel/8.3.2 - intel-classic/2021.4.0 + - node.js/20.10.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' @@ -31,6 +32,7 @@ compilers:: modules: - PrgEnv-gnu/8.3.2 - gcc/10.3.0 + - node.js/20.10.0 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib' diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 7188d7b4e..cdeebff1e 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -115,6 +115,10 @@ packages: prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 modules: - qt/5.15.2 + sed: + externals: + - spec: sed@4.4 + prefix: /usr subversion: externals: - spec: subversion@1.10.6 From 53835dcf7e81af52e7c13a6ffd340ddcdc4a7b94 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 14:46:54 -0700 Subject: [PATCH 15/25] Add doc/modulefile_templates/node.js and update documentation for spack-stack-1.6.0 for Narwhal (Intel) and S4 --- doc/modulefile_templates/node.js | 23 +++++++++++++++++++++++ doc/source/MaintainersSection.rst | 22 +++++++++++++++++++++- doc/source/PreConfiguredSites.rst | 16 +++++++++------- 3 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 doc/modulefile_templates/node.js diff --git a/doc/modulefile_templates/node.js b/doc/modulefile_templates/node.js new file mode 100644 index 000000000..7f5fc113c --- /dev/null +++ b/doc/modulefile_templates/node.js @@ -0,0 +1,23 @@ +#%Module1.0 + +module-whatis "Provides an node.js-20.10.0 installation for use with spack." + +# Only allow one instance of compiler to load +conflict ecflow + +proc ModulesHelp { } { +puts stderr "Provides an node.js-20.10.0 installation for use with spack." +} + +if { [ module-info mode load ] && ![ is-loaded gcc/10.3.0 ] } { + module load gcc/10.3.0 +} + +# Set this value +set NODEJS_PATH "/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0" + +prepend-path PATH "${NODEJS_PATH}/bin" +prepend-path LD_LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path CPATH "${NODEJS_PATH}/include" +prepend-path CMAKE_PREFIX_PATH "${NODEJS_PATH}" diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 5235641c1..5abfdaca7 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -346,7 +346,7 @@ mysql NAVY HPCMP Narwhal ------------------------------ -On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. +On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. Also, temporarily it is necessary to install ``node.js`` as an external package to work around build errors for ``py-jupyter-server`` (see https://github.com/JCSDA/spack-stack/issues/928 and https://github.com/spack/spack/issues/41899). git-lfs The following instructions install ``git-lfs`` in ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0``. Version 2.10.0 is the default version for Narwhal. First, download the ``git-lfs`` RPM on a system with full internet access (e.g., Derecho) using ``wget https://download.opensuse.org/repositories/openSUSE:/Leap:/15.2/standard/x86_64/git-lfs-2.10.0-lp152.1.2.x86_64.rpm`` and copy this file to ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0/src``. Then switch to Narwhal and run the following commands. @@ -397,6 +397,26 @@ ecflow mysql ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31``. +node.js + ``node.js`` is difficult to install via ``spack``, but is needed to install certain Python packages. The complication is that when using a newer ``gcc`` compiler (either directly or as backend for ``icc`` etc.), the OS ``node.js`` errors out with unresolved symbols in the ``libstdc++`` library. Therefore, we need to install ``node.js`` with ``gcc@10.3.0`` loaded, and create modulefile ``node.js/20.10.0`` from template ``modulefiles/node.js``. + +.. code-block:: console + + module unload PrgEnv-cray + module load PrgEnv-gnu/8.3.2 + module unload gcc + module load gcc/10.3.0 + + mkdir -p node-js-20.10.0/src && cd node-js-20.10.0/src + wget https://nodejs.org/dist/v20.10.0/node-v20.10.0.tar.gz + tar -xvzf node-v20.10.0.tar.gz + cd node-v20.10.0/ + ./configure --partly-static \ + --prefix=/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0 \ + 2>&1 | tee log.config + make 2>&1 | tee log.make + make install 2>&1 | tee log.install + .. _MaintainersSection_Nautilus: ------------------------------ diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e1b1bd990..ce1867cae 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -36,7 +36,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -44,7 +44,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Mark Potts | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | S4 | GCC | *currently not supported* | Dom Heinzeller / Mark Potts | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -204,15 +204,16 @@ With Intel, the following is required for building new spack environments and fo module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load node.js/20.10.0 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! @@ -233,6 +234,7 @@ With GNU, the following is required for building new spack environments and for module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load node.js/20.10.0 For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. @@ -549,11 +551,11 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.10.8 From fa309610cc35031739b95a70198371c68320ddd7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 15:17:31 -0700 Subject: [PATCH 16/25] More updates for Hercules (special installs) --- doc/source/PreConfiguredSites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ce1867cae..3e253e3a2 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -141,7 +141,7 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. note:: - spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. + spack-stack-1.6.0 on Hercules has additional packages `fms@2023.02.01`, `sp@2.3.0`, and `crtm@2.4.0` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. .. _Preconfigured_Sites_Discover: From 22f2cdd5c28d5834d85762a0413493f807015db9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 18:22:52 -0700 Subject: [PATCH 17/25] Update doc/source/PreConfiguredSites.rst for gsi-addon-env on S4 --- doc/source/PreConfiguredSites.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 3e253e3a2..e7146740d 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -558,13 +558,16 @@ For ``spack-stack-1.6.0`` with Intel, load the following modules after loading m module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 module unuse /opt/apps/modulefiles/Compiler/intel/22 module available Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. +.. note:: + spack-stack-1.6.0 on S4 provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. note:: There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. From 6d6dcdcd1d022838fc16c105426986395c22cc61 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jan 2024 15:14:37 -0700 Subject: [PATCH 18/25] release/1.6.0 site config & documentation updates for Orion, Derecho, Narwhal/GNU, AWS ParallelCluster, AWS Single-Node AMI Red Hat 8, Casper (#930) Summary Second round of site config and documentation updates for spack-stack-1.6.0: Orion, Derecho, Narwhal/GNU, AWS ParallelCluster, AWS Single-Node AMI Red Hat 8, Casper. Also, add information for a new GNU+OpenMPI environment on Hercules that is available for testing. Notes: - Bug in CDO package requires reverting back to 2.0.5 on Orion instead of the default 2.2.0 (see Installation issue: cdo@2.2.0 fails to build on systems that don't have a python3 interpreter in the default search path spack/spack#41947) - Problems with recently activated file locking mechanism in spack require turning off locking on AWS ParallelCluster (i.e. when the shared, slow EFS filesystem is used) - Submodule pointer update for spack for a container template bug fix (release/1.6.0: bugfix - repair dockerfile template spack#387) - The Derecho site config updates fix the problem with more and watch not being functional, see "more" does not work on Derecho with UWM spack-stack modules loaded #910 --- configs/sites/aws-pcluster/config.yaml | 9 ++- configs/sites/derecho/packages.yaml | 35 ++--------- configs/sites/hercules/packages.yaml | 11 ++++ configs/sites/orion/packages.yaml | 7 +-- doc/source/MaintainersSection.rst | 52 +++++++++++++++- doc/source/NewSiteConfigs.rst | 3 +- doc/source/PreConfiguredSites.rst | 86 +++++++++++++++----------- spack | 2 +- 8 files changed, 129 insertions(+), 76 deletions(-) diff --git a/configs/sites/aws-pcluster/config.yaml b/configs/sites/aws-pcluster/config.yaml index 3c8ac4c79..d9db1b9e1 100644 --- a/configs/sites/aws-pcluster/config.yaml +++ b/configs/sites/aws-pcluster/config.yaml @@ -1,10 +1,15 @@ config: - build_jobs: 24 + build_jobs: 8 # Overrides for spack build and staging areas to speed up builds # by using a local directory instead of the EFS shared filesystem - build_stage: /tmp/spack-stack/cache/build_stage test_stage: /tmp/spack-stack/cache/test_stage source_cache: /tmp/spack-stack/cache/source_cache misc_cache: /tmp/spack-stack/cache/misc_cache + + # Turn off file locking due to problems with EFS filesystem. + # Do not run parallel "spack install" commands on aws-pcluster! + db_lock_timeout: 60 + package_lock_timeout: 60 + locks: false diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml index f06ac3933..ec4dfb8ab 100644 --- a/configs/sites/derecho/packages.yaml +++ b/configs/sites/derecho/packages.yaml @@ -36,32 +36,22 @@ packages: ### All other external packages listed alphabetically autoconf: externals: - - spec: autoconf@2.71 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: autoconf@2.69 prefix: /usr automake: externals: - - spec: automake@1.16.5 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: automake@1.15.1 prefix: /usr binutils: externals: - spec: binutils@2.37.20211103 prefix: /usr - cmake: - externals: - - spec: cmake@3.26.3 - prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/cmake/3.26.3/gcc/7.5.0/k34x coreutils: externals: - spec: coreutils@8.32 prefix: /usr curl: externals: - - spec: curl@8.1.2+nghttp2 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: curl@7.79.1+gssapi+ldap+nghttp2 prefix: /usr cvs: @@ -83,10 +73,6 @@ packages: externals: - spec: findutils@4.8.0 prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /glade/u/apps/derecho/23.09/opt/view gawk: externals: - spec: gawk@4.2.1 @@ -94,13 +80,13 @@ packages: git: externals: - spec: git@2.41.0+tcltk - prefix: /glade/u/apps/derecho/23.09/opt/view + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git/2.41.0/gcc/7.5.0/jgni - spec: git@2.35.3+tcltk prefix: /usr git-lfs: externals: - spec: git-lfs@3.3.0 - prefix: /glade/u/apps/derecho/23.09/opt/view + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git-lfs/3.3.0/gcc/7.5.0/u3kh gmake: externals: - spec: gmake@4.2.1 @@ -113,21 +99,10 @@ packages: externals: - spec: krb5@1.19.2 prefix: /usr/lib/mit - libtool: - externals: - - spec: libtool@2.4.7 - prefix: /glade/u/apps/derecho/23.09/opt/view - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.18 prefix: /usr - # Automatically detected, but don't use - missing "ninja" dependency - #meson: - # externals: - # - spec: meson@1.2.0 - # prefix: /glade/u/apps/derecho/23.09/opt/view mysql: buildable: False externals: @@ -147,6 +122,10 @@ packages: externals: - spec: pkg-config@0.29.2 prefix: /usr + sed: + externals: + - spec: sed@4.4 + prefix: /usr subversion: externals: - spec: subversion@1.14.1 @@ -157,8 +136,6 @@ packages: prefix: /usr texinfo: externals: - - spec: texinfo@7.0.3 - prefix: /glade/u/apps/derecho/23.09/opt/view - spec: texinfo@6.5 prefix: /usr wget: diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 900b60913..b3deda8f5 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -3,6 +3,8 @@ packages: compiler:: [intel@2021.9.0, gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0, mvapich2@2.3.7] + # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing + #mpi:: [openmpi@4.1.6] ### MPI, Python, MKL mpi: @@ -20,6 +22,15 @@ packages: modules: - gcc/12.2.0 - mvapich2/2.3.7 + # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing + #openmpi: + # externals: + # - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath + # schedulers=slurm + # prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack + # modules: + # - gcc/12.2.0 + # - openmpi/4.1.6-gcc-12.2.0-spack ### Modifications of common packages diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 2a9ac00ec..e5b3f50be 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -30,6 +30,9 @@ packages: - openmpi/4.0.4 ### Modifications of common packages + # Temporary - see https://github.com/spack/spack/issues/41947 + cdo: + require: '@2.0.5' ### All other external packages listed alphabetically autoconf: @@ -125,10 +128,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 5abfdaca7..fd8002748 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -270,7 +270,7 @@ mysql ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules``. mvapich2 - Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2``. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler. + Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2`` outside of spack and provide it as an external package. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler. .. code-block:: console @@ -290,6 +290,54 @@ mvapich2 make check make install +openmpi (currently only for testing) + Because of difficulties with the default ``openmpi`` on Hercules, we build ``openmpi`` outside of spack and provide it as an external package. It is necessary to load the ``gcc`` compiler module and the ``zlib`` module for consistency. The configuration options are mostly adopted from the default OpenMPI installations that were done by the system administrators using spack (many of them are default values), except that we use internal ``hwloc`` and ``pmix``. Create modulefile ``openmpi`` from template ``doc/modulefile_templates/openmpi``. + +.. code-block:: console + + ./configure \ + --enable-shared \ + --disable-silent-rules \ + --disable-builtin-atomics \ + --with-pmi=/opt/slurm \ + --enable-static \ + --enable-mpi1-compatibility \ + --without-hcoll \ + --without-psm2 \ + --without-knem \ + --without-verbs \ + --without-psm \ + --without-cma \ + --without-ucx \ + --without-mxm \ + --without-fca \ + --without-xpmem \ + --without-ofi \ + --without-cray-xpmem \ + --without-sge \ + --without-lsf \ + --without-loadleveler \ + --without-alps \ + --without-tm \ + --with-slurm \ + --disable-memchecker \ + --with-pmix=internal \ + --with-zlib=/apps/spack-managed/gcc-12.2.0/zlib-1.2.13-p3sxbyfgvvjy7jx4kizib2jwvhm4s6l4 \ + --with-hwloc=internal \ + --disable-java \ + --disable-mpi-java \ + --with-gpfs=no \ + --without-cuda \ + --enable-wrapper-rpath \ + --disable-wrapper-runpath \ + --disable-mpi-cxx \ + --disable-cxx-exceptions \ + --with-wrapper-ldflags="-Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib/gcc/x86_64-pc-linux-gnu/12.2.0 -Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib64" \ + --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack 2>&1 | tee log.config + make VERBOSE=1 -j4 + make check + make install + .. _MaintainersSection_Discover: ------------------------------ @@ -621,7 +669,7 @@ mysql Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ -See ``configs/sites/aws-pcluster/README.md``. +See ``configs/sites/aws-pcluster/README.md``. It is important to note that because the shared EFS filesystem is slow and had problems with the spack locking mechanism introduced mid 2023, all file locking is turned off in the site config. Therefore, one must never run more than one ``spack install`` command for the same environment - that is, no parallel ``spack install`` processes. .. _MaintainersSection_Testing_New_Packages: diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index b8c39c421..783005b9a 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -453,6 +453,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack external find --scope system wget spack external find --scope system mysql spack external find --scope system texlive + spack external find --scope system sed 5. Find compilers, add to site config's ``compilers.yaml`` @@ -490,8 +491,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack config add "packages:fontconfig:variants:+pic" spack config add "packages:pixman:variants:+pic" spack config add "packages:cairo:variants:+pic" - spack config add "packages:libffi:version:['3.3']" - spack config add "packages:flex:version:['2.6.4']" 9. If you have manually installed lmod, you will need to update the site module configuration to use lmod instead of tcl. Skip this step if you followed the Ubuntu or Red Hat instructions above. diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e7146740d..da686001c 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -20,13 +20,15 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | ++---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ +| MSU | Hercules gcc+openmpi alternative | GCC | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416`` | Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Cam Book | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -38,7 +40,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -50,9 +52,9 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -83,26 +85,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available +.. note:: + The unified environment on Orion uses ``cdo@2.0.5`` instead of the default ``cdo@2.2.0`` because of a bug in the ``cdo`` package recipe that affects systems that don't have a ``python3`` interpreter in the default search paths (see https://github.com/spack/spack/issues/41947) for more information. This is a temporary change on Orion for the spack-stack-1.6.0 release and will be reverted once the ``cdo`` package is updated in the upstream spack develop code. + +.. note:: + spack-stack-1.6.0 on Orion provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + ------------------------------ MSU Hercules ------------------------------ @@ -137,6 +145,14 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys module load stack-python/3.10.13 module available +For testing, an additional version of spack-stack-1.6.0 with GNU+OpenMPI is available. Load the following modules after loading mysql and ecflow: + + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416/install/modulefiles/Core/ + module load stack-gcc/12.2.0 + module load stack-openmpi/4.1.6 + module load stack-python/3.10.13 + module available + .. note:: spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. @@ -204,7 +220,6 @@ With Intel, the following is required for building new spack environments and fo module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 - module load node.js/20.10.0 For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. @@ -234,16 +249,15 @@ With GNU, the following is required for building new spack environments and for module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 - module load node.js/20.10.0 -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.10.8 + module load stack-python/3.10.13 .. _Preconfigured_Sites_Nautilus: @@ -321,15 +335,15 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Derecho: @@ -350,24 +364,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.33 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading ecflow and mysql: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading ecflow and mysql: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading ecflow and mysql: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading ecflow and mysql: .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. note:: @@ -578,27 +592,27 @@ Amazon Web Services Parallelcluster Ubuntu 20.04 Access to the JCSDA-managed AWS Parallel Clusters is not available to the public. The following instructions are for JCSDA core staff and in-kind contributors. -For ``spack-stack-1.5.1`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: +For ``spack-stack-1.6.0`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: .. code-block:: console module purge ulimit -s unlimited source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh - module use /mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.1.0 module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: +For ``spack-stack-1.6.0`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: module purge ulimit -s unlimited - module use /mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v7/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.4.0 module load stack-openmpi/4.1.4 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. note:: @@ -609,18 +623,18 @@ For ``spack-stack-1.5.1`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` Amazon Web Services Red Hat 8 ----------------------------- -Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-6.1.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-7.1.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). -For ``spack-stack-1.5.1``, run: +For ``spack-stack-1.6.0``, run: .. code-block:: console ulimit -s unlimited scl_source enable gcc-toolset-11 - module use /home/ec2-user/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.2.1 module load stack-openmpi/4.1.5 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Configurable_Sites_CreateEnv: diff --git a/spack b/spack index 46e4875a0..76bf231c0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 46e4875a0b7021094f0ddabae74e087c64849641 +Subproject commit 76bf231c08097be24a7a851f5bdca1ca15c1f53b From c259e6479ceb9f8900e02799388f3496af00286b Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Mon, 8 Jan 2024 22:32:52 -0500 Subject: [PATCH 19/25] Remove libtool form Gaea, Hera and Jet. (#937) --- configs/sites/gaea-c5/packages.yaml | 4 ---- configs/sites/hera/packages.yaml | 4 ---- configs/sites/jet/packages.yaml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/configs/sites/gaea-c5/packages.yaml b/configs/sites/gaea-c5/packages.yaml index 306fce195..0c7032604 100644 --- a/configs/sites/gaea-c5/packages.yaml +++ b/configs/sites/gaea-c5/packages.yaml @@ -122,10 +122,6 @@ packages: prefix: /usr libtirpc: variants: ~gssapi - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr # This package is currently incomplete (no headers), but still works libxaw: externals: diff --git a/configs/sites/hera/packages.yaml b/configs/sites/hera/packages.yaml index 75ec8fd4a..95c8e4e42 100644 --- a/configs/sites/hera/packages.yaml +++ b/configs/sites/hera/packages.yaml @@ -124,10 +124,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/configs/sites/jet/packages.yaml b/configs/sites/jet/packages.yaml index 5f16ef38b..5ae9b49a6 100644 --- a/configs/sites/jet/packages.yaml +++ b/configs/sites/jet/packages.yaml @@ -129,10 +129,6 @@ packages: externals: - spec: libfuse@2.9.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 From d6010bdf3b912a82675b05500b79a387fe964c1c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 10 Jan 2024 12:54:35 -0700 Subject: [PATCH 20/25] release/1.6.0: last site config/doc updates and spack submodule update (release/1.6.0 --> spack-stack-1.6.0) (#935) * Update spack submodule from branch release/1.6.0 to tag spack-stack-1.6.0 * Update documentation for acorn * Update documentation for spack-stack-1.6.0 for Hera, Jet, Gaea C5 --- .gitmodules | 4 +- .../containers/docker-ubuntu-clang-mpich.yaml | 2 +- .../containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- .../containers/docker-ubuntu-intel-impi.yaml | 2 +- doc/source/PreConfiguredSites.rst | 47 +++++++++++-------- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17f5bc5fc..1a6d0b7fb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop url = https://github.com/jcsda/spack - branch = release/1.6.0 + branch = spack-stack-1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index c31a1ac6d..5dd9bb517 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.6.0 + ref: spack-stack-1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index f7120b680..15cb87c16 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -97,7 +97,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.6.0 + ref: spack-stack-1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 99f29349b..5436b6d71 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.6.0 + ref: spack-stack-1.6.0 resolve_sha: true # Whether or not to strip binaries diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index da686001c..e0b55d61b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -30,13 +30,13 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Cam Book | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Hang Lei / Alex Richert | +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.1/envs/unified-env`` | Alex Richert / Dom Heinzeller | +| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -393,7 +393,7 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading ecf NOAA Acorn (WCOSS2 test system) ------------------------------- -For spack-stack-1.5.1, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core``. +For spack-stack-1.6.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core``. On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! @@ -456,14 +456,14 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2023.1.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module -t available .. note:: @@ -492,28 +492,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.5.3 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. +.. note:: + + spack-stack-1.6.0 on Hera provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. _Preconfigured_Sites_Jet: ------------------------------ @@ -531,26 +535,30 @@ The following is required for building new spack environments and for using spac module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.8 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 module load stack-python/3.10.8 module available +.. note:: + + spack-stack-1.6.0 on Jet provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ @@ -580,6 +588,7 @@ For ``spack-stack-1.6.0`` with Intel, load the following modules after loading m Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. .. note:: + spack-stack-1.6.0 on S4 provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. note:: @@ -617,7 +626,7 @@ For ``spack-stack-1.6.0`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` .. note:: - The GNU stack is currently under testing and may not work as expected. We recommend using the well-tested Intel setup. + spack-stack-1.6.0 on AWS ParallelCluster has an additional package ``metis@5.1.0`` installed for both Intel and GNU. This is needed for MPAS, but must be loaded explictly (i.e. not part of ``jedi-mpas-env`` yet). ----------------------------- Amazon Web Services Red Hat 8 From 23a81419e767815c8cfa80c0c373ecb4d78b7f37 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 12 Jan 2024 09:25:30 -0700 Subject: [PATCH 21/25] release/1.6.0: fix bug in documentation for Hercules, site config updates/cleanup for ParallelWorks (#943) * Fix bug in documentation for Hercules * Changes for NOAA Azure, remove use of intel/2018 compiler on NOAA platforms --------- Co-authored-by: Natalie Perlin --- configs/sites/noaa-aws/compilers.yaml | 14 -------------- configs/sites/noaa-aws/packages.yaml | 8 -------- configs/sites/noaa-azure/compilers.yaml | 14 -------------- configs/sites/noaa-azure/packages.yaml | 12 ------------ configs/sites/noaa-gcloud/compilers.yaml | 14 -------------- configs/sites/noaa-gcloud/packages.yaml | 8 -------- configs/templates/unified-dev/spack.yaml | 8 -------- doc/source/PreConfiguredSites.rst | 4 +++- 8 files changed, 3 insertions(+), 79 deletions(-) diff --git a/configs/sites/noaa-aws/compilers.yaml b/configs/sites/noaa-aws/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-aws/compilers.yaml +++ b/configs/sites/noaa-aws/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-aws/packages.yaml b/configs/sites/noaa-aws/packages.yaml index 1cbe78222..3af5a9560 100644 --- a/configs/sites/noaa-aws/packages.yaml +++ b/configs/sites/noaa-aws/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 diff --git a/configs/sites/noaa-azure/compilers.yaml b/configs/sites/noaa-azure/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-azure/compilers.yaml +++ b/configs/sites/noaa-azure/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-azure/packages.yaml b/configs/sites/noaa-azure/packages.yaml index 886727adf..6cec5293b 100644 --- a/configs/sites/noaa-azure/packages.yaml +++ b/configs/sites/noaa-azure/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 @@ -50,10 +42,6 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - cmake: - externals: - - spec: cmake@3.27.2 - prefix: /contrib/spack-stack/cmake-3.27.2 cpio: externals: - spec: cpio@2.11 diff --git a/configs/sites/noaa-gcloud/compilers.yaml b/configs/sites/noaa-gcloud/compilers.yaml index 7a2b91b60..aec6336cf 100644 --- a/configs/sites/noaa-gcloud/compilers.yaml +++ b/configs/sites/noaa-gcloud/compilers.yaml @@ -16,20 +16,6 @@ compilers: PATH: '/apps/gnu/gcc-9.2.0/bin' LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' extra_rpaths: [] -- compiler: - spec: intel@18.0.5 - paths: - cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc - cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc - f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 - target: x86_64 - modules: - - intel/18.0.5.274 - environment: {} - extra_rpaths: [] - compiler: spec: gcc@9.2.0 paths: diff --git a/configs/sites/noaa-gcloud/packages.yaml b/configs/sites/noaa-gcloud/packages.yaml index b3abc7a21..7a7215bd7 100644 --- a/configs/sites/noaa-gcloud/packages.yaml +++ b/configs/sites/noaa-gcloud/packages.yaml @@ -1,10 +1,8 @@ packages: all: compiler:: [intel@2021.3.0, gcc@9.2.0] - #compiler:: [intel@18.0.5] providers: mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4] - #mpi:: [intel-mpi@2018.4.274] ### MPI, Python, MKL mpi: @@ -15,12 +13,6 @@ packages: prefix: /apps/oneapi modules: - impi/2021.3.0 - intel-mpi: - externals: - - spec: intel-mpi@2018.4.274%intel@18.0.5 - prefix: /apps/intel/compilers_and_libraries_2018 - modules: - - impi/2018.4.274 openmpi: externals: - spec: openmpi@3.1.4%gcc@9.2.0 diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 54d6c1e69..8baebc82b 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -48,11 +48,3 @@ spack: exclude: # jedi-tools doesn't build with Intel - jedi-tools-env%intel - ## Don't even bother building those with Intel 18, - ## not needed. Many of the Python packages don't build. - #- ewok-env%intel@18 - #- jedi-fv3-env%intel@18 - #- jedi-tools-env%intel@18 - #- jedi-ufs-env%intel@18 - #- jedi-um-env%intel@18 - #- soca-env%intel@18 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e0b55d61b..947d39ffa 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -147,7 +147,9 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys For testing, an additional version of spack-stack-1.6.0 with GNU+OpenMPI is available. Load the following modules after loading mysql and ecflow: - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416/install/modulefiles/Core/ +.. code-block:: console + + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-openmpi/4.1.6 module load stack-python/3.10.13 From e6ed7ae27fe99ba951b7ac0643a8394e9a02ab26 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 13 Jan 2024 20:55:39 -0700 Subject: [PATCH 22/25] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 6 +++++- spack | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1a6d0b7fb..3e2cc7000 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,11 @@ [submodule "spack"] path = spack + #url = https://github.com/spack/spack + #branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack url = https://github.com/jcsda/spack - branch = spack-stack-1.6.0 + branch = feature/merge_rel160_into_develop [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 76bf231c0..71adcea84 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 76bf231c08097be24a7a851f5bdca1ca15c1f53b +Subproject commit 71adcea84a89819ead3e0f233bc6751d773c85db From 390ad62f5fea4ed7591c32ef44e1fcbd8f4077af Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 13 Jan 2024 21:00:25 -0700 Subject: [PATCH 23/25] Revert changes that were only needed on the release/1.6.0 branch --- .gitmodules | 1 - configs/containers/docker-ubuntu-clang-mpich.yaml | 2 +- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- configs/sites/aws-pcluster/config.yaml | 6 ------ doc/source/MaintainersSection.rst | 2 +- doc/source/conf.py | 4 ++-- 7 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.gitmodules b/.gitmodules index 3e2cc7000..f4985d2e2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,6 @@ #branch = develop #url = https://github.com/jcsda/spack #branch = jcsda_emc_spack_stack - url = https://github.com/jcsda/spack branch = feature/merge_rel160_into_develop [submodule "doc/CMakeModules"] path = doc/CMakeModules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 5dd9bb517..fd475d055 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: spack-stack-1.6.0 + ref: jcsda_emc_spack_stack resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 15cb87c16..7c2ca47b8 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -97,7 +97,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: spack-stack-1.6.0 + ref: jcsda_emc_spack_stack resolve_sha: true # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 5436b6d71..abc283707 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: spack-stack-1.6.0 + ref: jcsda_emc_spack_stack resolve_sha: true # Whether or not to strip binaries diff --git a/configs/sites/aws-pcluster/config.yaml b/configs/sites/aws-pcluster/config.yaml index d9db1b9e1..cf54e6360 100644 --- a/configs/sites/aws-pcluster/config.yaml +++ b/configs/sites/aws-pcluster/config.yaml @@ -7,9 +7,3 @@ config: test_stage: /tmp/spack-stack/cache/test_stage source_cache: /tmp/spack-stack/cache/source_cache misc_cache: /tmp/spack-stack/cache/misc_cache - - # Turn off file locking due to problems with EFS filesystem. - # Do not run parallel "spack install" commands on aws-pcluster! - db_lock_timeout: 60 - package_lock_timeout: 60 - locks: false diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index fd8002748..6cf8a46dc 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -669,7 +669,7 @@ mysql Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ -See ``configs/sites/aws-pcluster/README.md``. It is important to note that because the shared EFS filesystem is slow and had problems with the spack locking mechanism introduced mid 2023, all file locking is turned off in the site config. Therefore, one must never run more than one ``spack install`` command for the same environment - that is, no parallel ``spack install`` processes. +See ``configs/sites/aws-pcluster/README.md``. .. _MaintainersSection_Testing_New_Packages: diff --git a/doc/source/conf.py b/doc/source/conf.py index 0558e8818..ca4ccb5ff 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,10 +24,10 @@ author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' # The short X.Y version -version = '1.6' +version = 'dev' # The full version, including alpha/beta/rc tags -release = '1.6.0' +release = 'develop' numfig = True From 83e7ec1c7f3ad4a4826f930e6b06d328d760b278 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 13 Jan 2024 21:19:34 -0700 Subject: [PATCH 24/25] Fix typo in .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index f4985d2e2..d1234e604 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,7 @@ path = spack #url = https://github.com/spack/spack #branch = develop - #url = https://github.com/jcsda/spack + url = https://github.com/jcsda/spack #branch = jcsda_emc_spack_stack branch = feature/merge_rel160_into_develop [submodule "doc/CMakeModules"] From dbc25776fc43420013067fd4cd1a445bf1d12a02 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 14 Jan 2024 16:03:59 -0700 Subject: [PATCH 25/25] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 3 +-- spack | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index d1234e604..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,8 +3,7 @@ #url = https://github.com/spack/spack #branch = develop url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - branch = feature/merge_rel160_into_develop + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 71adcea84..2d27c7383 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 71adcea84a89819ead3e0f233bc6751d773c85db +Subproject commit 2d27c7383b6ece7ce8ba5df57defdebd7180ea16