release/1.6.0 site config & documentation updates for Orion, Derecho, Narwhal/GNU, AWS ParallelCluster, AWS Single-Node AMI Red Hat 8, Casper

configs/sites/aws-pcluster/config.yaml

@@ -1,10 +1,15 @@
 config:
-  build_jobs: 24
+  build_jobs: 8
 
   # Overrides for spack build and staging areas to speed up builds
   # by using a local directory instead of the EFS shared filesystem
-
   build_stage: /tmp/spack-stack/cache/build_stage
   test_stage: /tmp/spack-stack/cache/test_stage
   source_cache: /tmp/spack-stack/cache/source_cache
   misc_cache: /tmp/spack-stack/cache/misc_cache
+
+  # Turn off file locking due to problems with EFS filesystem.
+  # Do not run parallel "spack install" commands on aws-pcluster!
+  db_lock_timeout: 60
+  package_lock_timeout: 60
+  locks: false

configs/sites/derecho/packages.yaml

@@ -36,32 +36,22 @@ packages:
 ### All other external packages listed alphabetically
   autoconf:
     externals:
-    - spec: autoconf@2.71
-      prefix: /glade/u/apps/derecho/23.09/opt/view
     - spec: autoconf@2.69
       prefix: /usr
   automake:
     externals:
-    - spec: automake@1.16.5
-      prefix: /glade/u/apps/derecho/23.09/opt/view
     - spec: automake@1.15.1
       prefix: /usr
   binutils:
     externals:
     - spec: binutils@2.37.20211103
       prefix: /usr
-  cmake:
-    externals:
-    - spec: cmake@3.26.3
-      prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/cmake/3.26.3/gcc/7.5.0/k34x
   coreutils:
     externals:
     - spec: coreutils@8.32
       prefix: /usr
   curl:
     externals:
-    - spec: curl@8.1.2+nghttp2
-      prefix: /glade/u/apps/derecho/23.09/opt/view
     - spec: curl@7.79.1+gssapi+ldap+nghttp2
       prefix: /usr
   cvs:
@@ -83,24 +73,20 @@ packages:
     externals:
     - spec: findutils@4.8.0
       prefix: /usr
-  flex:
-    externals:
-    - spec: flex@2.6.4+lex
-      prefix: /glade/u/apps/derecho/23.09/opt/view
   gawk:
     externals:
     - spec: gawk@4.2.1
       prefix: /usr
   git:
     externals:
     - spec: git@2.41.0+tcltk
-      prefix: /glade/u/apps/derecho/23.09/opt/view
+      prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git/2.41.0/gcc/7.5.0/jgni
     - spec: git@2.35.3+tcltk
       prefix: /usr
   git-lfs:
     externals:
     - spec: git-lfs@3.3.0
-      prefix: /glade/u/apps/derecho/23.09/opt/view
+      prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git-lfs/3.3.0/gcc/7.5.0/u3kh
   gmake:
     externals:
     - spec: gmake@4.2.1
@@ -113,21 +99,10 @@ packages:
     externals:
     - spec: krb5@1.19.2
       prefix: /usr/lib/mit
-  libtool:
-    externals:
-    - spec: libtool@2.4.7
-      prefix: /glade/u/apps/derecho/23.09/opt/view
-    - spec: libtool@2.4.6
-      prefix: /usr
   m4:
     externals:
     - spec: m4@1.4.18
       prefix: /usr
-  # Automatically detected, but don't use - missing "ninja" dependency
-  #meson:
-  #  externals:
-  #  - spec: meson@1.2.0
-  #    prefix: /glade/u/apps/derecho/23.09/opt/view
   mysql:
     buildable: False
     externals:
@@ -147,6 +122,10 @@ packages:
     externals:
     - spec: pkg-config@0.29.2
       prefix: /usr
+  sed:
+    externals:
+    - spec: sed@4.4
+      prefix: /usr
   subversion:
     externals:
     - spec: subversion@1.14.1
@@ -157,8 +136,6 @@ packages:
       prefix: /usr
   texinfo:
     externals:
-    - spec: texinfo@7.0.3
-      prefix: /glade/u/apps/derecho/23.09/opt/view
     - spec: texinfo@6.5
       prefix: /usr
   wget:

configs/sites/hercules/packages.yaml

@@ -3,6 +3,8 @@ packages:
     compiler:: [intel@2021.9.0, gcc@12.2.0]
     providers:
       mpi:: [intel-oneapi-mpi@2021.9.0, mvapich2@2.3.7]
+      # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing
+      #mpi:: [openmpi@4.1.6]
 
 ### MPI, Python, MKL
   mpi:
@@ -20,6 +22,15 @@ packages:
       modules:
       - gcc/12.2.0
       - mvapich2/2.3.7
+  # Alternative openmpi@4.1.6 with gcc@12.2.0 for testing
+  #openmpi:
+  #  externals:
+  #  - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
+  #      schedulers=slurm
+  #    prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack
+  #    modules:
+  #    - gcc/12.2.0
+  #    - openmpi/4.1.6-gcc-12.2.0-spack
 
 ### Modifications of common packages
 

configs/sites/orion/packages.yaml

@@ -30,6 +30,9 @@ packages:
       - openmpi/4.0.4
 
 ### Modifications of common packages
+  # Temporary - see https://github.com/spack/spack/issues/41947
+  cdo:
+    require: '@2.0.5'
 
 ### All other external packages listed alphabetically
   autoconf:
@@ -125,10 +128,6 @@ packages:
     externals:
     - spec: libfuse@2.9.2
       prefix: /usr
-  libtool:
-    externals:
-    - spec: libtool@2.4.2
-      prefix: /usr
   libxpm:
     externals:
     - spec: libxpm@4.11.0

doc/source/MaintainersSection.rst

@@ -270,7 +270,7 @@ mysql
   ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules``.
 
 mvapich2
-  Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2``. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler.
+  Because of difficulties with ``openmpi`` on Hercules, we build ``mvapich2`` outside of spack and provide it as an external package. It is necessary to either load ``qt`` to use a consistent ``zlib``, or to load ``zlib`` directly (check the ``qt`` module). Create modulefile ``mvapich2`` from template ``doc/modulefile_templates/mvapich2``. **Important:** We identified a bug in ``gcc@11`` + ``mvapich2@2.3.7`` in MPI allgather operations. It is therefore necessary to switch to a newer GCC compiler.
 
 .. code-block:: console
 
@@ -290,6 +290,54 @@ mvapich2
    make check
    make install
 
+openmpi (currently only for testing)
+  Because of difficulties with the default ``openmpi`` on Hercules, we build ``openmpi`` outside of spack and provide it as an external package. It is necessary to load the ``gcc`` compiler module and the ``zlib`` module for consistency. The configuration options are mostly adopted from the default OpenMPI installations that were done by the system administrators using spack (many of them are default values), except that we use internal ``hwloc`` and ``pmix``. Create modulefile ``openmpi`` from template ``doc/modulefile_templates/openmpi``.
+
+.. code-block:: console
+
+   ./configure \
+       --enable-shared \
+       --disable-silent-rules \
+       --disable-builtin-atomics \
+       --with-pmi=/opt/slurm \
+       --enable-static \
+       --enable-mpi1-compatibility \
+       --without-hcoll \
+       --without-psm2 \
+       --without-knem \
+       --without-verbs \
+       --without-psm \
+       --without-cma \
+       --without-ucx \
+       --without-mxm \
+       --without-fca \
+       --without-xpmem \
+       --without-ofi \
+       --without-cray-xpmem \
+       --without-sge \
+       --without-lsf \
+       --without-loadleveler \
+       --without-alps \
+       --without-tm \
+       --with-slurm \
+       --disable-memchecker \
+       --with-pmix=internal \
+       --with-zlib=/apps/spack-managed/gcc-12.2.0/zlib-1.2.13-p3sxbyfgvvjy7jx4kizib2jwvhm4s6l4 \
+       --with-hwloc=internal \
+       --disable-java \
+       --disable-mpi-java \
+       --with-gpfs=no \
+       --without-cuda \
+       --enable-wrapper-rpath \
+       --disable-wrapper-runpath \
+       --disable-mpi-cxx \
+       --disable-cxx-exceptions \
+       --with-wrapper-ldflags="-Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib/gcc/x86_64-pc-linux-gnu/12.2.0 -Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib64" \
+       --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack 2>&1 | tee log.config
+   make VERBOSE=1 -j4
+   make check
+   make install
+
 .. _MaintainersSection_Discover:
 
 ------------------------------
@@ -621,7 +669,7 @@ mysql
 Amazon Web Services Parallelcluster Ubuntu 20.04
 ------------------------------------------------
 
-See ``configs/sites/aws-pcluster/README.md``.
+See ``configs/sites/aws-pcluster/README.md``. It is important to note that because the shared EFS filesystem is slow and had problems with the spack locking mechanism introduced mid 2023, all file locking is turned off in the site config. Therefore, one must never run more than one ``spack install`` command for the same environment - that is, no parallel ``spack install`` processes.
 
 .. _MaintainersSection_Testing_New_Packages:
 

doc/source/NewSiteConfigs.rst

@@ -453,6 +453,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
    spack external find --scope system wget
    spack external find --scope system mysql
    spack external find --scope system texlive
+   spack external find --scope system sed
 
 5. Find compilers, add to site config's ``compilers.yaml``
 
@@ -490,8 +491,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
    spack config add "packages:fontconfig:variants:+pic"
    spack config add "packages:pixman:variants:+pic"
    spack config add "packages:cairo:variants:+pic"
-   spack config add "packages:libffi:version:['3.3']"
-   spack config add "packages:flex:version:['2.6.4']"
 
 9. If you have manually installed lmod, you will need to update the site module configuration to use lmod instead of tcl. Skip this step if you followed the Ubuntu or Red Hat instructions above.