From 073f814ba00fb85070cf696b18b9a1908f755391 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 16 Nov 2023 13:07:14 -0800 Subject: [PATCH 01/68] Set up submodule for testing --- .gitmodules | 4 ++-- spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..befaa3fc7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + url = https://github.com/AlexanderRichert-NOAA/spack + branch = nov2023_spackmerge [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 4ac3371a1..8522a647a 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 4ac3371a1274e79f7d83e26cfd755a14ac9f628a +Subproject commit 8522a647a35a4c26d3de4f9801ef15f5e35a041b From 963460bb746ead299518c965259ceea0f9cd9048 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 16 Nov 2023 15:45:28 -0800 Subject: [PATCH 02/68] try removing openssl default version --- configs/common/packages.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index d68b16614..479c6d3bf 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -189,7 +189,6 @@ variants: +internal-hwloc +two_level_namespace # Pin openssl to avoid duplicate packages being built openssl: - version: ['1.1.1u'] variants: +shared p4est: version: ['2.8'] From 9a6c6bfcd80ed011e08c259f6fb5726833c3453c Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 16 Nov 2023 16:24:04 -0800 Subject: [PATCH 03/68] Remove some defaults to clear up duplicates --- configs/common/packages.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 479c6d3bf..09daf29e5 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -263,12 +263,6 @@ version: ['0.12.2'] py-scipy: version: ['1.9.3'] - # Pin the py-setuptools version to avoid duplicate Python packages - py-setuptools: - version: ['59.4.0'] - # Pin the py-setuptools-scm version to avoid duplicate Python packages - py-setuptools-scm: - version: ['7.0.5'] py-shapely: version: ['1.8.0'] qt: From 4cc3cf342514dfc79c2bf7c06b73fac4851f63db Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 16 Nov 2023 17:26:51 -0800 Subject: [PATCH 04/68] update default prod-util --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 09daf29e5..29b0cd2c4 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -203,7 +203,7 @@ pkgconf: buildable: False prod-util: - version: ['1.2.2'] + version: ['2.1.0'] proj: version: ['8.1.0'] variants: ~tiff From 662cb3e2f7c61523b220924fe445c44b78be54bb Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 16 Nov 2023 17:45:55 -0800 Subject: [PATCH 05/68] use new deprecated_io variant name for fms --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 29b0cd2c4..c9ee89d83 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -82,7 +82,7 @@ #version: ['2023.01'] #variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release version: ['2023.02.01'] - variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +use_fmsio + variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io fontconfig: variants: +pic freetype: From 0b8f0767435630de99741e584780f30b44f26469 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 20 Nov 2023 14:29:30 -0800 Subject: [PATCH 06/68] submodule update --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 8522a647a..678d9545a 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 8522a647a35a4c26d3de4f9801ef15f5e35a041b +Subproject commit 678d9545a3debb86a8cfdfb1babf4af4fd04cac0 From e29b15f8849b47e21b21982a0f55a025f7fc1929 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 20 Nov 2023 19:29:54 -0800 Subject: [PATCH 07/68] submodule pointer (libdap4 update) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 678d9545a..2a7688402 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 678d9545a3debb86a8cfdfb1babf4af4fd04cac0 +Subproject commit 2a768840208ddcfeda2cc6a38b26615155c28a3f From 6408d4a097e10f0730d838ba3b55e92bcebce22b Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 07:45:53 -0800 Subject: [PATCH 08/68] submodule pointer (libxpm update) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 2a7688402..50c75ad98 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 2a768840208ddcfeda2cc6a38b26615155c28a3f +Subproject commit 50c75ad98d98081f5ee24f48d407d46897d53442 From 377881d04208f53c36348271eb0de693a5713c5e Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 07:47:33 -0800 Subject: [PATCH 09/68] submodule pointer --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 50c75ad98..673383e6c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 50c75ad98d98081f5ee24f48d407d46897d53442 +Subproject commit 673383e6ccc58e4761ab2936fb63b44698bfd306 From 7cc9b16fd6ab4ac717fdb59247a65a7913a31b9f Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 07:55:55 -0800 Subject: [PATCH 10/68] add _8 precision variants for w3emc, sp, ip --- configs/common/packages.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index c9ee89d83..82220eb42 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -119,6 +119,7 @@ variants: +hl +fortran +mpi ~threadsafe +szip ip: version: ['4.3.0'] + variants: precision=4,d,8 ip2: version: ['1.1.2'] jasper: @@ -278,12 +279,14 @@ version: ['2.3.2'] sp: version: ['2.3.3'] + variants: precision=4,d,8 udunits: version: ['2.2.28'] upp: version: ['10.0.10'] w3emc: version: ['2.10.0'] + variants: precision=4,d,8 w3nco: version: ['2.4.1'] wget: From 1fbc6770bd31d0f181aeffb6fdf1441f3b0f421e Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 07:56:20 -0800 Subject: [PATCH 11/68] submodule pointer --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 673383e6c..f0f8eda76 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 673383e6ccc58e4761ab2936fb63b44698bfd306 +Subproject commit f0f8eda76b76992b66ff1c608d2d5ede46d5a6e1 From a7aabe7a7079bf2a53a34206ecccff222e2182d5 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 07:57:37 -0800 Subject: [PATCH 12/68] submodule pointer --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index f0f8eda76..afeb98d43 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit f0f8eda76b76992b66ff1c608d2d5ede46d5a6e1 +Subproject commit afeb98d431941f44ba9cbfc9928c725d8680e90d From 68da01f724e44cfa94f1feecd5101190f1d25a0a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 08:27:53 -0800 Subject: [PATCH 13/68] submodule pointer (gmake fix) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index afeb98d43..4b436283c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit afeb98d431941f44ba9cbfc9928c725d8680e90d +Subproject commit 4b436283c4139423cf65a400d7edbed8f20e5098 From c3e7d5c420f8d20b808b6f52e15349abc88c4357 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 09:39:37 -0800 Subject: [PATCH 14/68] mysql: set download_boost as default --- configs/common/packages.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 82220eb42..be0af8890 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -153,6 +153,8 @@ version: ['5.1.0'] mpich: variants: ~hwloc +two_level_namespace + mysql: + variants: +download_boost nco: version: ['5.0.6'] variants: ~doc From 8d336594fa1fd2ffc7003d6c718a94c63ceee3d6 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 13:06:38 -0800 Subject: [PATCH 15/68] submodule pointer (mysql update) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 4b436283c..5944caacd 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 4b436283c4139423cf65a400d7edbed8f20e5098 +Subproject commit 5944caacd86cc519b72581c314f2da43482e48d6 From 4525443b8de7fa8812372c0c95d67b1555595228 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 14:11:58 -0800 Subject: [PATCH 16/68] submodule pointer (merge jcsda_emc_spack_stack) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 5944caacd..7e492ed14 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5944caacd86cc519b72581c314f2da43482e48d6 +Subproject commit 7e492ed14d24eff3f035bb83bfec2eb0a9ecf874 From 458572a020cd50affe044927d5d91e8c9cf39b30 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 14:14:19 -0800 Subject: [PATCH 17/68] fiddling with submodule --- .gitmodules | 4 ++-- spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index befaa3fc7..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/AlexanderRichert-NOAA/spack - branch = nov2023_spackmerge + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 7e492ed14..42fbcfb1a 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7e492ed14d24eff3f035bb83bfec2eb0a9ecf874 +Subproject commit 42fbcfb1af07d09e30469ac1d37979a64f25ce19 From f146a26ee04a210ef0fa9f4ed1e5778110036a5d Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 14:16:40 -0800 Subject: [PATCH 18/68] point submodule back to test repo --- .gitmodules | 4 ++-- spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..befaa3fc7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + url = https://github.com/AlexanderRichert-NOAA/spack + branch = nov2023_spackmerge [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 42fbcfb1a..7e492ed14 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 42fbcfb1af07d09e30469ac1d37979a64f25ce19 +Subproject commit 7e492ed14d24eff3f035bb83bfec2eb0a9ecf874 From b04cdf0abbc847401f2a3ee15d11bee2add0ce1e Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 14:41:03 -0800 Subject: [PATCH 19/68] submodule pointer (lmod.py fix) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 7e492ed14..90843a15d 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7e492ed14d24eff3f035bb83bfec2eb0a9ecf874 +Subproject commit 90843a15d792dc474515d7a33aeb441b631fdf02 From d4963700b04e2b0b1957ea5708af1670c3ce7bdf Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 14:55:12 -0800 Subject: [PATCH 20/68] submodule pointer (containers __init__.py fix) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 90843a15d..c608921ff 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 90843a15d792dc474515d7a33aeb441b631fdf02 +Subproject commit c608921ff36d60cb1f09512414416995afe7a65b From aef9bbb5f48b7280a83e2dadd3ee231bd52aa2ef Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 15:58:34 -0800 Subject: [PATCH 21/68] pin gettext version because of intel (ubuntu CI) issue (one-arg static_assert) --- configs/common/packages.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 3306c06ef..a72424c9c 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -93,6 +93,8 @@ version: ['1.6.4'] g2tmpl: version: ['1.10.2'] + gettext: + version: ['0.21.1'] gfsio: version: ['1.4.1'] gftl-shared: From ee7a96af51e68857207e67a9724bc010c32d6af9 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 16:06:01 -0800 Subject: [PATCH 22/68] fix macos-ci-aarch64.yaml (buildcache build-env call) --- .github/workflows/macos-ci-aarch64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 74177f928..b93935ed9 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -105,7 +105,7 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ + spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env # the rest echo "${{ inputs.template || 'unified-dev' }} ..." From d8e18a183182f6f58a1a380cbe9d30ce778561e9 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 21 Nov 2023 16:42:59 -0800 Subject: [PATCH 23/68] fix ci (buildcache calls) --- .github/workflows/macos-ci-x86_64.yaml | 2 +- .github/workflows/ubuntu-ci-x86_64.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 9fa39abb3..2c8762154 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -98,7 +98,7 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ + spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env # the rest echo "${{ inputs.template || 'unified-dev' }} ..." diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index 88b1b9584..5afece5c1 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -136,7 +136,7 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2022.1.0.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env # the rest echo "${{ inputs.template || 'unified-dev' }} ..." From ea55f96ea08b84e9105fab4084a408ea8c037e31 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 14:49:22 -0700 Subject: [PATCH 24/68] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index c608921ff..c17971e9a 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c608921ff36d60cb1f09512414416995afe7a65b +Subproject commit c17971e9a4e07f3d4331797e1c5e1373980c108e From db71994ca8dac033958b51d71c94a260ef5a88c4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 16:12:33 -0700 Subject: [PATCH 25/68] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index c17971e9a..c1b6bd541 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c17971e9a4e07f3d4331797e1c5e1373980c108e +Subproject commit c1b6bd5415f6cad49b61768417b122883031b3bd From 3a4c991bf988737ceda2038a0d1fac785cad5d71 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 17:03:27 -0700 Subject: [PATCH 26/68] Trigger CI From 63a608c75254f45cf45321c14939404633c0d7cd Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:24:52 -0800 Subject: [PATCH 27/68] submodule pointer (revert prebuilt aws; add rust-bootstrap for py-cryptography) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index c1b6bd541..6669ab04d 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c1b6bd5415f6cad49b61768417b122883031b3bd +Subproject commit 6669ab04d957cc0a4386bae76cab53a30e75eca6 From c3cce380d742f0f5c39a7588fe8ce10be2511971 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:29:17 -0800 Subject: [PATCH 28/68] set py-cryptography+rust_bootstrap by default --- configs/common/packages.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index a72424c9c..5552c7699 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -194,6 +194,7 @@ variants: +internal-hwloc +two_level_namespace # Pin openssl to avoid duplicate packages being built openssl: + require: ['@1.1.1u'] variants: +shared p4est: version: ['2.8'] @@ -225,7 +226,8 @@ version: ['8.0.3'] # Pin py-cryptography to a version earlier than 3.4 to avoid the rust dependency py-cryptography: - version: ['3.3.2'] + require: ['@3.3.2'] + variants: +rust_bootstrap py-eccodes: version: ['1.4.2'] py-h5py: From 88a181fe4417ec609ec0a0072b0577762cac01ef Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:34:05 -0800 Subject: [PATCH 29/68] remove openssl version setting --- configs/common/packages.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 5552c7699..69ba10dd4 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -194,7 +194,6 @@ variants: +internal-hwloc +two_level_namespace # Pin openssl to avoid duplicate packages being built openssl: - require: ['@1.1.1u'] variants: +shared p4est: version: ['2.8'] From e0879632d393e94f8c7430e6d97f026d10be4680 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:34:26 -0800 Subject: [PATCH 30/68] submodule pointer --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 6669ab04d..c09cd7d45 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6669ab04d957cc0a4386bae76cab53a30e75eca6 +Subproject commit c09cd7d450deb0b5e88885fc447d7516e2e6adfc From df51b6645b7ad6616dcbe6b38261dc56ab6be02c Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:39:11 -0800 Subject: [PATCH 31/68] remove py-crypto version setting --- configs/common/packages.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 69ba10dd4..70b2342cf 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -223,9 +223,7 @@ variants: +plotting py-click: version: ['8.0.3'] - # Pin py-cryptography to a version earlier than 3.4 to avoid the rust dependency py-cryptography: - require: ['@3.3.2'] variants: +rust_bootstrap py-eccodes: version: ['1.4.2'] From e2e3b1d0465365978d19c46aaadb3aa252218376 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 15:45:27 -0800 Subject: [PATCH 32/68] use rust-bootstrap for py-setuptools-rust --- configs/common/packages.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 70b2342cf..ddcc393fb 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -267,6 +267,8 @@ version: ['0.12.2'] py-scipy: version: ['1.9.3'] + py-setuptools-rust: + variants: +rust_bootstrap py-shapely: version: ['1.8.0'] qt: From ab7b0cf98f62510219ac4364e55675a4c38a7cd8 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 16:38:09 -0800 Subject: [PATCH 33/68] submodule pointer --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index c09cd7d45..8d9cb63e7 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c09cd7d450deb0b5e88885fc447d7516e2e6adfc +Subproject commit 8d9cb63e7c5c9fb878908c594ebaef7952508363 From cedf95070d6fa48b6a1268a7c6ce9a3ad349be6a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 18:17:59 -0800 Subject: [PATCH 34/68] pin py-numpy version --- configs/common/packages.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ddcc393fb..172be5ec9 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -240,8 +240,7 @@ version: ['1.5.8'] variants: ~mpi py-numpy: - version: ['1.22.3'] - variants: +blas +lapack + require: ['@1.22.3'] py-pandas: variants: +excel # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer From 199a93b23f62bc123337908cecb48a5ca449c6de Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 18:19:01 -0800 Subject: [PATCH 35/68] test numpy build (revert this commit) --- .github/workflows/ubuntu-ci-x86_64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index 5afece5c1..bec60b15c 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -135,6 +135,7 @@ jobs: # base-env echo "base-env ..." + spack install --fail-fast --no-check-signature py-numpy spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2022.1.0.base-env spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env From 2f9b718293bdf7a12083f39d271d5255e6b1253a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 18:31:39 -0800 Subject: [PATCH 36/68] Revert "test numpy build (revert this commit)" This reverts commit 199a93b23f62bc123337908cecb48a5ca449c6de. --- .github/workflows/ubuntu-ci-x86_64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index bec60b15c..5afece5c1 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -135,7 +135,6 @@ jobs: # base-env echo "base-env ..." - spack install --fail-fast --no-check-signature py-numpy spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2022.1.0.base-env spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env From e1cfc21c9e9cc506e012979fbe381d3ecc4ef52c Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 18:42:34 -0800 Subject: [PATCH 37/68] re-pin py-setuptools version --- configs/common/packages.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 172be5ec9..df4d820e3 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -266,6 +266,9 @@ version: ['0.12.2'] py-scipy: version: ['1.9.3'] + # Pin the py-setuptools version to avoid duplicate Python packages + py-setuptools: + require: ['@63.4.3'] py-setuptools-rust: variants: +rust_bootstrap py-shapely: From 9963f47c0b9efa2b39c1489f438b08951a3d43fb Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 22:08:53 -0800 Subject: [PATCH 38/68] debug (revert this commit) --- .github/workflows/macos-ci-aarch64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index b93935ed9..89a4b21e3 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -104,6 +104,7 @@ jobs: # base-env echo "base-env ..." + spack install --fail-fast --no-check-signature hdf-eos2 spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env From a57f031a0249a3abd7b4d725367e384cfdd800af Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 22:09:38 -0800 Subject: [PATCH 39/68] submodule pointer (hdf-eos2/apple-clang) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 8d9cb63e7..7b9def0b4 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 8d9cb63e7c5c9fb878908c594ebaef7952508363 +Subproject commit 7b9def0b472fe7861f8b8456a04bef10fb3fb6cc From e5588b4d6f53f65d2ac4bcd26aba0991ec873588 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 29 Nov 2023 22:17:10 -0800 Subject: [PATCH 40/68] Revert "debug (revert this commit)" This reverts commit 9963f47c0b9efa2b39c1489f438b08951a3d43fb. --- .github/workflows/macos-ci-aarch64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 89a4b21e3..b93935ed9 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -104,7 +104,6 @@ jobs: # base-env echo "base-env ..." - spack install --fail-fast --no-check-signature hdf-eos2 spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env From d9018562c4849d4d17c3cf037fb0e24e76b32b3a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 30 Nov 2023 10:24:46 -0800 Subject: [PATCH 41/68] submodule pointer (test hdf-eos2 tweak) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 7b9def0b4..e0162286c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7b9def0b472fe7861f8b8456a04bef10fb3fb6cc +Subproject commit e0162286c4d086c9e67e68d83990b17616cd31e5 From 793bad41098c83778e63792d2431d4f8aeecc2ab Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 30 Nov 2023 10:25:15 -0800 Subject: [PATCH 42/68] revert me --- .github/workflows/macos-ci-aarch64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index b93935ed9..095f21fb7 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -104,6 +104,7 @@ jobs: # base-env echo "base-env ..." + spack install --fail-fast --source --no-check-signature hdf-eos2 spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env From 65f8cceb8b1f004ba0f282b24eb28f65a1af42e8 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 30 Nov 2023 10:30:49 -0800 Subject: [PATCH 43/68] Revert "revert me" This reverts commit 793bad41098c83778e63792d2431d4f8aeecc2ab. --- .github/workflows/macos-ci-aarch64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 095f21fb7..b93935ed9 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -104,7 +104,6 @@ jobs: # base-env echo "base-env ..." - spack install --fail-fast --source --no-check-signature hdf-eos2 spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env From ebea1bb0f1b7629f7d5805aedc19d8eb1ba0ddca Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Dec 2023 06:57:02 -0700 Subject: [PATCH 44/68] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index b1baf5b70..9140b8c00 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b1baf5b700e098a45f59da5ab990e1ace6b8442b +Subproject commit 9140b8c009a88e96eca93d510568dea7eb6f15c7 From c8d7167c4dcee910e9ce7f6a5811d9a044f0905e Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 4 Dec 2023 16:21:17 -0800 Subject: [PATCH 45/68] update rhel-emc packages.yaml (remove bison,python,llvm) --- configs/sites/emc-rhel/packages.yaml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/configs/sites/emc-rhel/packages.yaml b/configs/sites/emc-rhel/packages.yaml index 811495c36..d9071c300 100644 --- a/configs/sites/emc-rhel/packages.yaml +++ b/configs/sites/emc-rhel/packages.yaml @@ -21,10 +21,6 @@ packages: externals: - spec: binutils@2.30.117 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -128,22 +124,6 @@ packages: externals: - spec: libtool@2.4.6 prefix: /usr - llvm: - externals: - - spec: llvm@14.0.6+clang~lld~lldb - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/clang - cxx: /usr/bin/clang++ - llvm-doe: - externals: - - spec: llvm-doe@14.0.6+clang~lld~lldb - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/clang - cxx: /usr/bin/clang++ m4: externals: - spec: m4@1.4.18 @@ -156,10 +136,6 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr - python: - externals: - - spec: python@3.9.13+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /usr rsync: externals: - spec: rsync@3.1.3 From 96c85aa79b750a33834643a1df53c76a6cbcb7c1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Dec 2023 20:25:19 -0700 Subject: [PATCH 46/68] Update variant ectrans for ecmwf-atlas to just trans --- configs/common/packages.yaml | 2 +- configs/containers/specs/jedi-ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index df4d820e3..a98ba2bbb 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -52,7 +52,7 @@ variants: linalg=eigen,lapack compression=lz4,bzip2 ecmwf-atlas: version: ['0.35.0'] - variants: +fckit +ectrans +tesselation +fftw + variants: +fckit +trans +tesselation +fftw ectrans: version: ['1.2.0'] variants: ~mkl +fftw diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 3cc655da3..eb53ea3a6 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -1,7 +1,7 @@ ### spack-stack-1.5.1 / skylab-6.1.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) specs: [base-env@1.0.0, jedi-base-env@1.0.0, ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.1, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, - eckit@1.24.4, ecmwf-atlas@0.35.0 +fckit +ectrans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, + eckit@1.24.4, ecmwf-atlas@0.35.0 +fckit +trans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, eigen@3.4.0, fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.2, gftl-shared@1.6.1, gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.0, ip@4.3.0, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, From 88edf674914662116ce3363aad17a23fbb3cc11a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 4 Dec 2023 19:26:33 -0800 Subject: [PATCH 47/68] set zlib to be the provider of zlib-api in common/packages.yaml --- configs/common/packages.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index a98ba2bbb..477bdb302 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -10,6 +10,7 @@ jpeg: [libjpeg-turbo] lapack: [openblas] yacc: [bison] + zlib-api: [zlib] # # This version of awscli goes with py-pyyaml@5.4.1 awscli: From 305414dc3f71213f766906a986aebdb4629f887d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Dec 2023 20:37:55 -0700 Subject: [PATCH 48/68] Bump sp to 2.5.0 --- configs/common/packages.yaml | 2 +- configs/containers/specs/jedi-ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index a98ba2bbb..403d263c4 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -285,7 +285,7 @@ sigio: version: ['2.3.2'] sp: - version: ['2.3.3'] + version: ['2.5.0'] variants: precision=4,d,8 udunits: version: ['2.2.28'] diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index eb53ea3a6..c97f23e5e 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -9,7 +9,7 @@ parallelio@2.5.10, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3, py-gitpython@3.1.27, py-h5py@3.7.0, py-numpy@1.22.3, py-pandas@1.5.3, py-pip, py-pyyaml@5.4.1, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, - sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.5.0, mapl@2.40.3, + sp@2.5.0, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.5.0, mapl@2.40.3, zlib@1.2.13, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port, awscli-v2@2.13.22, py-globus-cli@3.16.0] # Notes: From de4b0800050f27cb882d9f23556da9ab6b56a877 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Dec 2023 07:05:59 -0700 Subject: [PATCH 49/68] Update .gitmodules for code review and testing --- .gitmodules | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..ec49adbd6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,10 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = feature/geos_gcm_dependencies [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules From bad3b9b9ded9fc238ac79c111be16ac866ea2ee5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Dec 2023 09:00:07 -0700 Subject: [PATCH 50/68] S4 site config update: remove external perl --- configs/sites/s4/packages.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/configs/sites/s4/packages.yaml b/configs/sites/s4/packages.yaml index 606b8cc16..20ad01b68 100644 --- a/configs/sites/s4/packages.yaml +++ b/configs/sites/s4/packages.yaml @@ -144,10 +144,6 @@ packages: prefix: /data/prod/jedi/spack-stack/mysql-8.0.31 modules: - mysql/8.0.31 - perl: - externals: - - spec: perl@5.16.3~cpanm+shared+threads - prefix: /usr pkg-config: externals: - spec: pkg-config@0.27.1 From ba2743ba7aeb78ef294e7362415d9112a818b76a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 5 Dec 2023 11:55:13 -0800 Subject: [PATCH 51/68] update submodule pointer (met fix) --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 1b8756ccc..12e7527c5 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 1b8756ccc30d49a6c117b852a87ce7be83307f1b +Subproject commit 12e7527c5b61b68b1faf27c74383f3130d186b1f From f8c3ad5706c75c1fdfb07f34eac34f2385305317 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Dec 2023 13:59:04 -0700 Subject: [PATCH 52/68] Update configs/sites/nautilus/packages.yaml --- configs/sites/nautilus/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 85f82c909..233e6d993 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -10,7 +10,7 @@ packages: fftw-api:: [intel-oneapi-mkl] lapack:: [intel-oneapi-mkl] ectrans: - variants:: +mkl + variants:: +mkl ~fftw gsibec: variants:: +mkl From 6b78fc5f8214070839037d4ad977ed194e85c9b4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Dec 2023 14:02:09 -0700 Subject: [PATCH 53/68] Update doc/source/KnownIssues.rst --- doc/source/KnownIssues.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index 3392210dd..974f7ed83 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -31,6 +31,10 @@ General When using a role account to install spack-stack, it is sometimes necessary to run graphical applications such as the ``qt`` online installer. The following website describes in detail how this can be done: https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo/ +7. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh`` + + This error usually indicates that the wrong module type is used in the ``spack module ... refresh`` command. For example, the system is configured for ``lmod``, but the command used is ``spack module tcl refresh``. + ============================== MSU Hercules ============================== From 390cced61c7c1c7f36d6a07fd5d4df58adf9cd1c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Dec 2023 08:35:32 -0700 Subject: [PATCH 54/68] Update doc/source/KnownIssues.rst for Discover --- doc/source/KnownIssues.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index 974f7ed83..b168084b6 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -49,6 +49,10 @@ NASA Discover Discover's connection to the outside world can be very slow and spack sometimes aborts with fetch timeouts. Try again until it works, sometimes have to wait for a bit. +2. ``configure: error: cannot guess build type; you must specify one`` when building ``freetype`` or other packages that use configure scripts + + This can happen if a spack install is started in a ``screen`` session, because Discover puts the temporary data in directories like ``/gpfsm/dnb33/tdirs/login/discover13.29716.dheinzel``, which get wiped out after some time. Without ``screen``, this problem doesn't occur. + ============================== NOAA Parallel Works ============================== From 2362336d28569b3b3e2a799e4a1162862d2fe839 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Dec 2023 10:11:03 -0700 Subject: [PATCH 55/68] Pin patchelf version on Derecho to avoid build errors with Intel --- configs/sites/derecho/packages.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml index 0b6d107a0..f06ac3933 100644 --- a/configs/sites/derecho/packages.yaml +++ b/configs/sites/derecho/packages.yaml @@ -29,6 +29,9 @@ packages: # Need to force spack to build python with ssl variant when using older openssl from OS python: require: "+ssl" + # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho + patchelf: + version:: ['0.17.2'] ### All other external packages listed alphabetically autoconf: From 953ec467cf57a81c74da6c6e14f5188930d68f5e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Dec 2023 10:56:59 -0700 Subject: [PATCH 56/68] Don't use external perl for Linux --- doc/source/NewSiteConfigs.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 6a722a8c0..2487bd4b8 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -450,7 +450,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi --exclude bison --exclude cmake \ --exclude curl --exclude openssl \ --exclude openssh - spack external find --scope system perl spack external find --scope system wget spack external find --scope system mysql spack external find --scope system texlive From e062fa70e8a187cf365c2f737bc65860fdccb8d9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Dec 2023 08:51:37 -0700 Subject: [PATCH 57/68] Enable variant extdata2g for mapl, required by geos-gcm --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index b5608aeee..9da5d0ab2 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -143,7 +143,7 @@ # https://github.com/JCSDA/spack-stack/issues/769 # also: ... extdata2g segfault UFS? version: ['2.40.3'] - variants: ~shared ~extdata2g ~pflogger + variants: ~shared ~pflogger # If making changes here, also check the Discover site config and the CI workflows met: version: ['11.1.0'] From 7a92ab7a4173abfc0cb61bd6f6d4809a44d30d25 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Dec 2023 08:32:17 -0700 Subject: [PATCH 58/68] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 4 ++-- spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index befaa3fc7..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/AlexanderRichert-NOAA/spack - branch = nov2023_spackmerge + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 12e7527c5..8683cc4f1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 12e7527c5b61b68b1faf27c74383f3130d186b1f +Subproject commit 8683cc4f1a42329cab38d2fd607476e202ce50d1 From 006e67f7c34da6222ea963391ac22156d7de6bad Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Dec 2023 20:14:19 -0700 Subject: [PATCH 59/68] Fix variants for mapl in configs/common/packages.yaml --- configs/common/packages.yaml | 2 +- spack | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 31e79b730..4e48cd670 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -142,7 +142,7 @@ # https://github.com/JCSDA/spack-stack/issues/769 # also: ... extdata2g segfault UFS? version: ['2.40.3'] - variants: ~shared ~pflogger + variants: ~shared +pflogger ~f2py # If making changes here, also check the Discover site config and the CI workflows met: version: ['11.1.0'] diff --git a/spack b/spack index 738deceef..6b7f86acc 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 738deceef3b8e36858606e5c3a61d7ba81b73537 +Subproject commit 6b7f86acc79c359eaabd17e23c0ac7773259dcd5 From a66f168e129f2a788554bd85bda533b0da7c4147 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Dec 2023 08:31:01 -0700 Subject: [PATCH 60/68] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 6b7f86acc..5aa071a2e 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6b7f86acc79c359eaabd17e23c0ac7773259dcd5 +Subproject commit 5aa071a2ea2993c4583e21ce22cd269f188bf70d From f4b164a13725d6840bf1907ffae3200f5ee4d08c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 16:16:45 -0700 Subject: [PATCH 61/68] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 559ebdaee..798183eb7 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 559ebdaee62f613762d693a92657bae3728a4660 +Subproject commit 798183eb78432139122e8ebcc1e08dc9dcc194d8 From 560b9e2c57f5302b6b24cc31da3c415ed7659f49 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 16 Jan 2024 15:32:28 -0700 Subject: [PATCH 62/68] Re-enable shared build of ESMF, needed for GEOS --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index b4a46290d..f9f3937a5 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -61,7 +61,7 @@ # Also, check the acorn and derecho site configs which have esmf modifications. esmf: version: ['8.5.0'] - variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio + variants: ~xerces ~pnetcdf snapshot=none +shared +external-parallelio require: - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: "%intel" From debc291fc18e339c20538ba98c59c150e6040d40 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 16 Jan 2024 15:45:38 -0700 Subject: [PATCH 63/68] Add geos-gcm-env to the unified-dev and skylab-dev templates --- configs/templates/skylab-dev/spack.yaml | 1 + configs/templates/unified-dev/spack.yaml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 615157137..f959ea1c7 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -9,6 +9,7 @@ spack: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel'] - packages: - ewok-env +ecflow +cylc + - geos-gcm-env - jedi-fv3-env - jedi-mpas-env - jedi-neptune-env diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 8baebc82b..6b5c887ab 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -8,10 +8,11 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel'] - packages: + - ewok-env +ecflow +cylc + - geos-gcm-env - global-workflow-env - gmao-swell-env - gsi-env - - ewok-env +ecflow +cylc - jedi-fv3-env - jedi-mpas-env - jedi-neptune-env From 99d06be578449131ec19e622c17444c1a844a0a1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 16 Jan 2024 15:50:06 -0700 Subject: [PATCH 64/68] Also build shared mapl --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index f9f3937a5..51487dc61 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -142,7 +142,7 @@ # https://github.com/JCSDA/spack-stack/issues/769 # also: ... extdata2g segfault UFS? version: ['2.40.3'] - variants: ~shared +pflogger ~f2py + variants: +shared +pflogger ~f2py # If making changes here, also check the Discover site config and the CI workflows met: version: ['11.1.0'] From 3f2529cec56516b5462523d2ec1c40f1d038d749 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 17 Jan 2024 10:03:05 -0700 Subject: [PATCH 65/68] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 6 ++---- spack | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index ec49adbd6..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,10 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = feature/geos_gcm_dependencies + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index a7802cea9..b25f3ea5f 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit a7802cea97b5c0d9f539b802a9e16e521f593e2c +Subproject commit b25f3ea5f13ab6d1af9fbf7821a1369e9af3b345 From 1b0a363c67d7f9c7962645e63ae521a8f0fbb0bb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 17 Jan 2024 18:43:51 -0700 Subject: [PATCH 66/68] Again update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..8bb5afb71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = bugfix/esmf_pflogger_geos [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index b25f3ea5f..2d4531c8e 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b25f3ea5f13ab6d1af9fbf7821a1369e9af3b345 +Subproject commit 2d4531c8e6670f3f3e17297f502d5de81cc14e23 From 5e26435ea8979d707a16db1a465e08e99ca2d503 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 17 Jan 2024 20:27:13 -0700 Subject: [PATCH 67/68] Enable mpi variant for pflogger as a workaround for the pflogger build system --- .github/workflows/ubuntu-ci-x86_64.yaml | 3 +++ .github/workflows/ubuntu-rnd-x86_64.yaml | 6 ++++++ configs/sites/aws-pcluster/packages.yaml | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/.github/workflows/ubuntu-ci-x86_64.yaml b/.github/workflows/ubuntu-ci-x86_64.yaml index dace76e13..64281eb78 100644 --- a/.github/workflows/ubuntu-ci-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-x86_64.yaml @@ -116,6 +116,9 @@ jobs: # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + # https://github.com/spack/spack/issues/42137 + spack config add "packages:pflogger:variants:+mpi" + # Concretize and check for duplicates spack concretize 2>&1 | tee log.concretize.intel-2022.1.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2022.1.0 -i fms -i crtm -i esmf -i mapl diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml index 6b924468c..e11a0e9ac 100644 --- a/.github/workflows/ubuntu-rnd-x86_64.yaml +++ b/.github/workflows/ubuntu-rnd-x86_64.yaml @@ -64,6 +64,9 @@ jobs: # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + # https://github.com/spack/spack/issues/42137 + spack config add "packages:pflogger:variants:+mpi" + # Overrides for spack build and staging areas to speed up builds, # separate from the default site config locations to avoid conflicts spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage" @@ -130,6 +133,9 @@ jobs: # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + # https://github.com/spack/spack/issues/42137 + spack config add "packages:pflogger:variants:+mpi" + # Overrides for spack build and staging areas to speed up builds, # separate from the default site config locations to avoid conflicts spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage" diff --git a/configs/sites/aws-pcluster/packages.yaml b/configs/sites/aws-pcluster/packages.yaml index ee9922c6d..50830f9a6 100644 --- a/configs/sites/aws-pcluster/packages.yaml +++ b/configs/sites/aws-pcluster/packages.yaml @@ -30,6 +30,12 @@ packages: - libfabric-aws/1.16.0~amzn4.0 - openmpi/4.1.4 +### Modification of common packages + + # https://github.com/spack/spack/issues/42137 + pflogger: + variants: +mpi + ### All other external packages listed alphabetically autoconf: externals: From 95077b1a79336297fbb0f40c0b9609600cfd0f7a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 18 Jan 2024 11:35:54 -0700 Subject: [PATCH 68/68] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 8bb5afb71..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = bugfix/esmf_pflogger_geos + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 2d4531c8e..c7af00bb3 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 2d4531c8e6670f3f3e17297f502d5de81cc14e23 +Subproject commit c7af00bb3c6d78f18e3db1190e3d50662b49abcf